diff --git a/docs/en/references/data/database-patterns.md b/docs/en/references/data/database-patterns.md index c4745832b9..320fdc49cc 100644 --- a/docs/en/references/data/database-patterns.md +++ b/docs/en/references/data/database-patterns.md @@ -1,5 +1,35 @@ # Database Schema Guidelines +## Schema File Organization + +### Principles + +| Scenario | Approach | +| -------------------------------------- | ------------------- | +| Strongly related tables in same domain | Merge into one file | +| Core tables / Complex business logic | One file per table | +| Tables that may cross multiple domains | One file per table | + +### Decision Criteria + +**Merge when:** + +- Tables have strong foreign key relationships (e.g., many-to-many) +- Tables belong to the same business domain +- Tables are unlikely to evolve independently + +**Separate (one file per table) when:** + +- Core table with many fields and complex logic +- Has a dedicated Service layer counterpart +- May expand independently in the future + +### File Naming + +- **Single-table files**: named after the table export name (`message.ts` for `messageTable`, `topic.ts` for `topicTable`) +- **Multi-table files**: lowercase, named by domain (`tagging.ts` for `tagTable` + `entityTagTable`) +- **Helper utilities**: underscore prefix (`_columnHelpers.ts`) to indicate non-table definitions + ## Naming Conventions - **Table names**: Use **singular** form with snake_case (e.g., `topic`, `message`, `app_state`) @@ -8,19 +38,19 @@ ## Column Helpers -All helpers are exported from `./schemas/columnHelpers.ts`. +All helpers are exported from `./schemas/_columnHelpers.ts`. ### Primary Keys -| Helper | UUID Version | Use Case | -|--------|--------------|----------| -| `uuidPrimaryKey()` | v4 (random) | General purpose tables | +| Helper | UUID Version | Use Case | +| ------------------------- | ----------------- | ------------------------------------ | +| `uuidPrimaryKey()` | v4 (random) | General purpose tables | | `uuidPrimaryKeyOrdered()` | v7 (time-ordered) | Large tables with time-based queries | **Usage:** ```typescript -import { uuidPrimaryKey, uuidPrimaryKeyOrdered } from './columnHelpers' +import { uuidPrimaryKey, uuidPrimaryKeyOrdered } from './_columnHelpers' // General purpose table export const topicTable = sqliteTable('topic', { @@ -45,29 +75,32 @@ export const messageTable = sqliteTable('message', { ### Timestamps -| Helper | Fields | Use Case | -|--------|--------|----------| -| `createUpdateTimestamps` | `createdAt`, `updatedAt` | Tables without soft delete | -| `createUpdateDeleteTimestamps` | `createdAt`, `updatedAt`, `deletedAt` | Tables with soft delete | +| Helper | Fields | Use Case | +| ------------------------------ | ------------------------------------- | -------------------------- | +| `createUpdateTimestamps` | `createdAt`, `updatedAt` | Tables without soft delete | +| `createUpdateDeleteTimestamps` | `createdAt`, `updatedAt`, `deletedAt` | Tables with soft delete | **Usage:** ```typescript -import { createUpdateTimestamps, createUpdateDeleteTimestamps } from './columnHelpers' +import { + createUpdateTimestamps, + createUpdateDeleteTimestamps, +} from "./_columnHelpers"; // Without soft delete -export const tagTable = sqliteTable('tag', { +export const tagTable = sqliteTable("tag", { id: uuidPrimaryKey(), name: text(), - ...createUpdateTimestamps -}) + ...createUpdateTimestamps, +}); // With soft delete -export const topicTable = sqliteTable('topic', { +export const topicTable = sqliteTable("topic", { id: uuidPrimaryKey(), name: text(), - ...createUpdateDeleteTimestamps -}) + ...createUpdateDeleteTimestamps, +}); ``` **Behavior:** @@ -81,7 +114,7 @@ export const topicTable = sqliteTable('topic', { For JSON column support, use `{ mode: 'json' }`: ```typescript -data: text({ mode: 'json' }).$type() +data: text({ mode: "json" }).$type(); ``` Drizzle handles JSON serialization/deserialization automatically. @@ -92,10 +125,10 @@ Drizzle handles JSON serialization/deserialization automatically. ```typescript // SET NULL: preserve record when referenced record is deleted -groupId: text().references(() => groupTable.id, { onDelete: 'set null' }) +groupId: text().references(() => groupTable.id, { onDelete: "set null" }); // CASCADE: delete record when referenced record is deleted -topicId: text().references(() => topicTable.id, { onDelete: 'cascade' }) +topicId: text().references(() => topicTable.id, { onDelete: "cascade" }); ``` ### Self-Referencing Foreign Keys @@ -103,23 +136,26 @@ topicId: text().references(() => topicTable.id, { onDelete: 'cascade' }) For self-referencing foreign keys (e.g., tree structures with parentId), **always use the `foreignKey` operator** in the table's third parameter: ```typescript -import { foreignKey, sqliteTable, text } from 'drizzle-orm/sqlite-core' +import { foreignKey, sqliteTable, text } from "drizzle-orm/sqlite-core"; export const messageTable = sqliteTable( - 'message', + "message", { id: uuidPrimaryKeyOrdered(), - parentId: text(), // Do NOT use .references() here + parentId: text(), // Do NOT use .references() here // ...other fields }, (t) => [ // Use foreignKey operator for self-referencing - foreignKey({ columns: [t.parentId], foreignColumns: [t.id] }).onDelete('set null') + foreignKey({ columns: [t.parentId], foreignColumns: [t.id] }).onDelete( + "set null" + ), ] -) +); ``` **Why this approach:** + - Avoids TypeScript circular reference issues (no need for `AnySQLiteColumn` type annotation) - More explicit and readable - Allows chaining `.onDelete()` / `.onUpdate()` actions @@ -142,21 +178,22 @@ If you encounter a scenario that seems to require circular references: ```typescript // ✅ GOOD: Break the cycle by handling one side at application layer -export const topicTable = sqliteTable('topic', { +export const topicTable = sqliteTable("topic", { id: uuidPrimaryKey(), // Application-managed reference (no FK constraint) // Validated by TopicService.setCurrentMessage() currentMessageId: text(), -}) +}); -export const messageTable = sqliteTable('message', { +export const messageTable = sqliteTable("message", { id: uuidPrimaryKeyOrdered(), // Database-enforced FK - topicId: text().references(() => topicTable.id, { onDelete: 'cascade' }), -}) + topicId: text().references(() => topicTable.id, { onDelete: "cascade" }), +}); ``` **Why soft references for SQLite:** + - SQLite does not support `DEFERRABLE` constraints (unlike PostgreSQL/Oracle) - Application-layer validation provides equivalent data integrity - Simplifies insert/update operations without transaction ordering concerns @@ -185,12 +222,12 @@ Always use `.returning()` to get inserted/updated data instead of re-querying: ```typescript // Good: Use returning() -const [row] = await db.insert(table).values(data).returning() -return rowToEntity(row) +const [row] = await db.insert(table).values(data).returning(); +return rowToEntity(row); // Avoid: Re-query after insert (unnecessary database round-trip) -await db.insert(table).values({ id, ...data }) -return this.getById(id) +await db.insert(table).values({ id, ...data }); +return this.getById(id); ``` ### Soft delete support diff --git a/src/main/data/db/DbService.ts b/src/main/data/db/DbService.ts index de72be03dd..a4a4db275d 100644 --- a/src/main/data/db/DbService.ts +++ b/src/main/data/db/DbService.ts @@ -6,7 +6,7 @@ import { app } from 'electron' import path from 'path' import { pathToFileURL } from 'url' -import { CUSTOM_SQL_STATEMENTS } from './customSql' +import { CUSTOM_SQL_STATEMENTS } from './customSqls' import Seeding from './seeding' import type { DbType } from './types' diff --git a/src/main/data/db/customSql.ts b/src/main/data/db/customSqls.ts similarity index 91% rename from src/main/data/db/customSql.ts rename to src/main/data/db/customSqls.ts index eaeea28db2..1db58bd3fd 100644 --- a/src/main/data/db/customSql.ts +++ b/src/main/data/db/customSqls.ts @@ -14,7 +14,7 @@ * 2. Import and spread them into CUSTOM_SQL_STATEMENTS below */ -import { MESSAGE_FTS_STATEMENTS } from './schemas/messageFts' +import { MESSAGE_FTS_STATEMENTS } from './schemas/message' /** * All custom SQL statements to run after migrations diff --git a/src/main/data/db/schemas/columnHelpers.ts b/src/main/data/db/schemas/_columnHelpers.ts similarity index 100% rename from src/main/data/db/schemas/columnHelpers.ts rename to src/main/data/db/schemas/_columnHelpers.ts diff --git a/src/main/data/db/schemas/appState.ts b/src/main/data/db/schemas/appState.ts index c64ccd95d0..9edadb77cf 100644 --- a/src/main/data/db/schemas/appState.ts +++ b/src/main/data/db/schemas/appState.ts @@ -1,6 +1,6 @@ import { sqliteTable, text } from 'drizzle-orm/sqlite-core' -import { createUpdateTimestamps } from './columnHelpers' +import { createUpdateTimestamps } from './_columnHelpers' export const appStateTable = sqliteTable('app_state', { key: text().primaryKey(), diff --git a/src/main/data/db/schemas/group.ts b/src/main/data/db/schemas/group.ts index 6ef06c522f..1acaf0d6bf 100644 --- a/src/main/data/db/schemas/group.ts +++ b/src/main/data/db/schemas/group.ts @@ -1,6 +1,6 @@ import { index, integer, sqliteTable, text } from 'drizzle-orm/sqlite-core' -import { createUpdateTimestamps, uuidPrimaryKey } from './columnHelpers' +import { createUpdateTimestamps, uuidPrimaryKey } from './_columnHelpers' /** * Group table - general-purpose grouping for entities diff --git a/src/main/data/db/schemas/message.ts b/src/main/data/db/schemas/message.ts index 3118433f6c..7b482a6ff4 100644 --- a/src/main/data/db/schemas/message.ts +++ b/src/main/data/db/schemas/message.ts @@ -3,7 +3,7 @@ import type { AssistantMeta, ModelMeta } from '@shared/data/types/meta' import { sql } from 'drizzle-orm' import { check, foreignKey, index, integer, sqliteTable, text } from 'drizzle-orm/sqlite-core' -import { createUpdateDeleteTimestamps, uuidPrimaryKeyOrdered } from './columnHelpers' +import { createUpdateDeleteTimestamps, uuidPrimaryKeyOrdered } from './_columnHelpers' import { topicTable } from './topic' /** @@ -63,3 +63,92 @@ export const messageTable = sqliteTable( check('message_status_check', sql`${t.status} IN ('pending', 'success', 'error', 'paused')`) ] ) + +/** + * FTS5 SQL statements for message full-text search + * + * This file contains SQL statements that must be manually added to migration files. + * Drizzle does not auto-generate virtual tables or triggers. + * + * Architecture: + * 1. message.searchable_text - regular column populated by trigger + * 2. message_fts - FTS5 virtual table with external content + * 3. Triggers sync both searchable_text and FTS5 index + * + * Usage: + * - Copy MESSAGE_FTS_MIGRATION_SQL to migration file when generating migrations + */ + +/** + * Custom SQL statements that Drizzle cannot manage + * These are executed after every migration via DbService.runCustomMigrations() + * + * All statements should use IF NOT EXISTS to be idempotent. + */ +export const MESSAGE_FTS_STATEMENTS: string[] = [ + // FTS5 virtual table, Links to message table's searchable_text column + `CREATE VIRTUAL TABLE IF NOT EXISTS message_fts USING fts5( + searchable_text, + content='message', + content_rowid='rowid', + tokenize='trigram' + )`, + + // Trigger: populate searchable_text and sync FTS on INSERT + `CREATE TRIGGER IF NOT EXISTS message_ai AFTER INSERT ON message BEGIN + UPDATE message SET searchable_text = ( + SELECT group_concat(json_extract(value, '$.content'), ' ') + FROM json_each(json_extract(NEW.data, '$.blocks')) + WHERE json_extract(value, '$.type') = 'main_text' + ) WHERE id = NEW.id; + INSERT INTO message_fts(rowid, searchable_text) + SELECT rowid, searchable_text FROM message WHERE id = NEW.id; + END`, + + // Trigger: sync FTS on DELETE + `CREATE TRIGGER IF NOT EXISTS message_ad AFTER DELETE ON message BEGIN + INSERT INTO message_fts(message_fts, rowid, searchable_text) + VALUES ('delete', OLD.rowid, OLD.searchable_text); + END`, + + // Trigger: update searchable_text and sync FTS on UPDATE OF data + `CREATE TRIGGER IF NOT EXISTS message_au AFTER UPDATE OF data ON message BEGIN + INSERT INTO message_fts(message_fts, rowid, searchable_text) + VALUES ('delete', OLD.rowid, OLD.searchable_text); + UPDATE message SET searchable_text = ( + SELECT group_concat(json_extract(value, '$.content'), ' ') + FROM json_each(json_extract(NEW.data, '$.blocks')) + WHERE json_extract(value, '$.type') = 'main_text' + ) WHERE id = NEW.id; + INSERT INTO message_fts(rowid, searchable_text) + SELECT rowid, searchable_text FROM message WHERE id = NEW.id; + END` +] + +/** Examples */ + +/** + * SQL expression to extract searchable text from data.blocks + * Concatenates content from all main_text type blocks + */ +// export const SEARCHABLE_TEXT_EXPRESSION = ` +// (SELECT group_concat(json_extract(value, '$.content'), ' ') +// FROM json_each(json_extract(NEW.data, '$.blocks')) +// WHERE json_extract(value, '$.type') = 'main_text') +// ` + +/** + * Rebuild FTS index (run manually if needed) + */ +// export const REBUILD_FTS_SQL = `INSERT INTO message_fts(message_fts) VALUES ('rebuild')` + +/** + * Example search query + */ +// export const EXAMPLE_SEARCH_SQL = ` +// SELECT m.* +// FROM message m +// JOIN message_fts fts ON m.rowid = fts.rowid +// WHERE message_fts MATCH ? +// ORDER BY rank +// ` diff --git a/src/main/data/db/schemas/messageFts.ts b/src/main/data/db/schemas/messageFts.ts deleted file mode 100644 index ccffbb5eaf..0000000000 --- a/src/main/data/db/schemas/messageFts.ts +++ /dev/null @@ -1,86 +0,0 @@ -/** - * FTS5 SQL statements for message full-text search - * - * This file contains SQL statements that must be manually added to migration files. - * Drizzle does not auto-generate virtual tables or triggers. - * - * Architecture: - * 1. message.searchable_text - regular column populated by trigger - * 2. message_fts - FTS5 virtual table with external content - * 3. Triggers sync both searchable_text and FTS5 index - * - * Usage: - * - Copy MESSAGE_FTS_MIGRATION_SQL to migration file when generating migrations - */ - -/** - * SQL expression to extract searchable text from data.blocks - * Concatenates content from all main_text type blocks - */ -export const SEARCHABLE_TEXT_EXPRESSION = ` - (SELECT group_concat(json_extract(value, '$.content'), ' ') - FROM json_each(json_extract(NEW.data, '$.blocks')) - WHERE json_extract(value, '$.type') = 'main_text') -` - -/** - * Custom SQL statements that Drizzle cannot manage - * These are executed after every migration via DbService.runCustomMigrations() - * - * All statements should use IF NOT EXISTS to be idempotent. - */ -export const MESSAGE_FTS_STATEMENTS: string[] = [ - // FTS5 virtual table, Links to message table's searchable_text column - `CREATE VIRTUAL TABLE IF NOT EXISTS message_fts USING fts5( - searchable_text, - content='message', - content_rowid='rowid', - tokenize='trigram' - )`, - - // Trigger: populate searchable_text and sync FTS on INSERT - `CREATE TRIGGER IF NOT EXISTS message_ai AFTER INSERT ON message BEGIN - UPDATE message SET searchable_text = ( - SELECT group_concat(json_extract(value, '$.content'), ' ') - FROM json_each(json_extract(NEW.data, '$.blocks')) - WHERE json_extract(value, '$.type') = 'main_text' - ) WHERE id = NEW.id; - INSERT INTO message_fts(rowid, searchable_text) - SELECT rowid, searchable_text FROM message WHERE id = NEW.id; - END`, - - // Trigger: sync FTS on DELETE - `CREATE TRIGGER IF NOT EXISTS message_ad AFTER DELETE ON message BEGIN - INSERT INTO message_fts(message_fts, rowid, searchable_text) - VALUES ('delete', OLD.rowid, OLD.searchable_text); - END`, - - // Trigger: update searchable_text and sync FTS on UPDATE OF data - `CREATE TRIGGER IF NOT EXISTS message_au AFTER UPDATE OF data ON message BEGIN - INSERT INTO message_fts(message_fts, rowid, searchable_text) - VALUES ('delete', OLD.rowid, OLD.searchable_text); - UPDATE message SET searchable_text = ( - SELECT group_concat(json_extract(value, '$.content'), ' ') - FROM json_each(json_extract(NEW.data, '$.blocks')) - WHERE json_extract(value, '$.type') = 'main_text' - ) WHERE id = NEW.id; - INSERT INTO message_fts(rowid, searchable_text) - SELECT rowid, searchable_text FROM message WHERE id = NEW.id; - END` -] - -/** - * Rebuild FTS index (run manually if needed) - */ -export const REBUILD_FTS_SQL = `INSERT INTO message_fts(message_fts) VALUES ('rebuild')` - -/** - * Example search query - */ -export const EXAMPLE_SEARCH_SQL = ` -SELECT m.* -FROM message m -JOIN message_fts fts ON m.rowid = fts.rowid -WHERE message_fts MATCH ? -ORDER BY rank -` diff --git a/src/main/data/db/schemas/preference.ts b/src/main/data/db/schemas/preference.ts index 5ca9b2f14a..9d7665fd53 100644 --- a/src/main/data/db/schemas/preference.ts +++ b/src/main/data/db/schemas/preference.ts @@ -1,6 +1,6 @@ import { primaryKey, sqliteTable, text } from 'drizzle-orm/sqlite-core' -import { createUpdateTimestamps } from './columnHelpers' +import { createUpdateTimestamps } from './_columnHelpers' export const preferenceTable = sqliteTable( 'preference', diff --git a/src/main/data/db/schemas/tag.ts b/src/main/data/db/schemas/tag.ts deleted file mode 100644 index 87820fadf9..0000000000 --- a/src/main/data/db/schemas/tag.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { sqliteTable, text } from 'drizzle-orm/sqlite-core' - -import { createUpdateTimestamps, uuidPrimaryKey } from './columnHelpers' - -/** - * Tag table - general-purpose tags for entities - * - * Tags can be applied to topics, sessions, and assistants - * via the entity_tag join table. - */ -export const tagTable = sqliteTable('tag', { - id: uuidPrimaryKey(), - // Unique tag name - name: text().notNull().unique(), - // Display color (hex code) - color: text(), - ...createUpdateTimestamps -}) diff --git a/src/main/data/db/schemas/entityTag.ts b/src/main/data/db/schemas/tagging.ts similarity index 64% rename from src/main/data/db/schemas/entityTag.ts rename to src/main/data/db/schemas/tagging.ts index e041d771db..a64c087951 100644 --- a/src/main/data/db/schemas/entityTag.ts +++ b/src/main/data/db/schemas/tagging.ts @@ -1,7 +1,21 @@ import { index, primaryKey, sqliteTable, text } from 'drizzle-orm/sqlite-core' -import { createUpdateTimestamps } from './columnHelpers' -import { tagTable } from './tag' +import { createUpdateTimestamps, uuidPrimaryKey } from './_columnHelpers' + +/** + * Tag table - general-purpose tags for entities + * + * Tags can be applied to topics, sessions, and assistants + * via the entity_tag join table. + */ +export const tagTable = sqliteTable('tag', { + id: uuidPrimaryKey(), + // Unique tag name + name: text().notNull().unique(), + // Display color (hex code) + color: text(), + ...createUpdateTimestamps +}) /** * Entity-Tag join table - associates tags with entities diff --git a/src/main/data/db/schemas/topic.ts b/src/main/data/db/schemas/topic.ts index 68078d8f86..09d1409a50 100644 --- a/src/main/data/db/schemas/topic.ts +++ b/src/main/data/db/schemas/topic.ts @@ -1,7 +1,7 @@ import type { AssistantMeta } from '@shared/data/types/meta' import { index, integer, sqliteTable, text } from 'drizzle-orm/sqlite-core' -import { createUpdateDeleteTimestamps, uuidPrimaryKey } from './columnHelpers' +import { createUpdateDeleteTimestamps, uuidPrimaryKey } from './_columnHelpers' import { groupTable } from './group' /**