♻️ refactor: move database to packages (#8874)

* move db

* refactor db import

* refactor eval types

* fix tests

* fix tests

* fix tests

* fix db migration issues

* fix docker issue

* fix tests

* update alias

* fix tests

* update db test for client and server

* refactor db

* update codecov

* update codecov

* update codecov

* add docker pr comments

.github/scripts/docker-pr-comment.js

@@ -0,0 +1,89 @@
+/**
+ * Generate or update PR comment with Docker build info
+ */
+module.exports = async ({
+  github,
+  context,
+  dockerMetaJson,
+  image,
+  version,
+  dockerhubUrl,
+  platforms,
+}) => {
+  const COMMENT_IDENTIFIER = '<!-- DOCKER-BUILD-COMMENT -->';
+
+  const parseTags = () => {
+    try {
+      if (dockerMetaJson) {
+        const parsed = JSON.parse(dockerMetaJson);
+        if (Array.isArray(parsed.tags) && parsed.tags.length > 0) {
+          return parsed.tags;
+        }
+      }
+    } catch (e) {
+      // ignore parsing error, fallback below
+    }
+    if (image && version) {
+      return [`${image}:${version}`];
+    }
+    return [];
+  };
+
+  const generateCommentBody = () => {
+    const tags = parseTags();
+    const buildTime = new Date().toISOString();
+
+    // Use the first tag as the main version
+    const mainTag = tags.length > 0 ? tags[0] : `${image}:${version}`;
+    const tagVersion = mainTag.includes(':') ? mainTag.split(':')[1] : version;
+
+    return [
+      COMMENT_IDENTIFIER,
+      '',
+      '### 🐳 Database Docker Build Completed!',
+      `**Version**: \`${tagVersion || 'N/A'}\``,
+      `**Build Time**: \`${buildTime}\``,
+      '',
+      dockerhubUrl ? `🔗 View all tags on Docker Hub: ${dockerhubUrl}` : '',
+      '',
+      '### Pull Image',
+      'Download the Docker image to your local machine:',
+      '',
+      '```bash',
+      `docker pull ${mainTag}`,
+      '```',
+      '> [!IMPORTANT]',
+      '> This build is for testing and validation purposes.',
+    ]
+      .filter(Boolean)
+      .join('\n');
+  };
+
+  const body = generateCommentBody();
+
+  // List comments on the PR
+  const { data: comments } = await github.rest.issues.listComments({
+    issue_number: context.issue.number,
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+  });
+
+  const existing = comments.find((c) => c.body && c.body.includes(COMMENT_IDENTIFIER));
+  if (existing) {
+    await github.rest.issues.updateComment({
+      comment_id: existing.id,
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      body,
+    });
+    return { updated: true, id: existing.id };
+  }
+
+  const result = await github.rest.issues.createComment({
+    issue_number: context.issue.number,
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    body,
+  });
+  return { updated: false, id: result.data.id };
+};

.github/workflows/docker-database.yml

@@ -159,3 +159,24 @@ jobs:
       - name: Inspect image
         run: |
           docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
+
+
+      - name: Comment on PR with Docker build info
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const prComment = require('${{ github.workspace }}/.github/scripts/docker-pr-comment.js');
+            const result = await prComment({
+              github,
+              context,
+              dockerMetaJson: ${{ toJSON(steps.meta.outputs.json) }},
+              image: "${{ env.REGISTRY_IMAGE }}",
+              version: "${{ steps.meta.outputs.version }}",
+              dockerhubUrl: "https://hub.docker.com/r/${{ env.REGISTRY_IMAGE }}/tags",
+              platforms: "linux/amd64, linux/arm64",
+            });
+            core.info(`Status: ${result.updated ? 'Updated' : 'Created'}, ID: ${result.id}`);
+
+

.github/workflows/test.yml

@@ -75,7 +75,7 @@ jobs:
           files: ./coverage/app/lcov.info
           flags: app
 
-  test-server:
+  test-databsae:
     name: Test Database
 
     runs-on: ubuntu-latest
@@ -109,8 +109,10 @@ jobs:
       - name: Lint
         run: bun run lint
 
-      - name: Test Server Coverage
-        run: bun run test-server:coverage
+      - uses: pnpm/action-setup@v4
+
+      - name: Test Coverage
+        run: pnpm --filter @lobechat/database test:coverage
         env:
           DATABASE_TEST_URL: postgresql://postgres:postgres@localhost:5432/postgres
           DATABASE_DRIVER: node
@@ -119,9 +121,9 @@ jobs:
           S3_PUBLIC_DOMAIN: https://example.com
           APP_URL: https://home.com
 
-      - name: Upload Server coverage to Codecov
+      - name: Upload Database coverage to Codecov
         uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./coverage/server/lcov.info
-          flags: server
+          files: ./packages/database/coverage/lcov.info
+          flags: database

Dockerfile.database

@@ -120,7 +120,7 @@ COPY --from=base /distroless/ /
 COPY --from=builder /app/.next/standalone /app/
 
 # Copy database migrations
-COPY --from=builder /app/src/database/migrations /app/migrations
+COPY --from=builder /app/packages/database/migrations /app/migrations
 COPY --from=builder /app/scripts/migrateServerDB/docker.cjs /app/docker.cjs
 COPY --from=builder /app/scripts/migrateServerDB/errorHint.js /app/errorHint.js
 

codecov.yml

@@ -1,11 +1,39 @@
+component_management:
+  individual_components:
+    # App architecture layers
+    - component_id: app_store
+      name: "Store"
+      paths:
+        - src/store/**
+    - component_id: app_services
+      name: "Services"
+      paths:
+        - src/services/**
+    - component_id: app_server
+      name: "Server"
+      paths:
+        - src/server/**
+    - component_id: app_libs
+      name: "Libs"
+      paths:
+        - src/libs/**
+    - component_id: app_utils
+      name: "Utils"
+      paths:
+        - src/utils/**
+
 coverage:
   status:
     project:
       default: off
-      server:
+      database:
         flags:
-          - server
+          - database
       app:
         flags:
           - app
     patch: off
+
+
+comment:
+  layout: "header, diff, flags, components"  # show component info in the PR comment

drizzle.config.ts

@@ -22,8 +22,8 @@ export default {
     url: connectionString,
   },
   dialect: 'postgresql',
-  out: './src/database/migrations',
+  out: './packages/database/migrations',
 
-  schema: './src/database/schemas',
+  schema: './packages/database/src/schemas',
   strict: true,
 } satisfies Config;

package.json

@@ -73,8 +73,6 @@
     "test": "npm run test-app && npm run test-server",
     "test-app": "vitest run --config vitest.config.ts",
     "test-app:coverage": "vitest run --config vitest.config.ts  --coverage",
-    "test-server": "vitest run --config vitest.config.server.ts",
-    "test-server:coverage": "vitest run --config vitest.config.server.ts --coverage",
     "test:update": "vitest -u",
     "type-check": "tsgo --noEmit",
     "webhook:ngrok": "ngrok http http://localhost:3011",
@@ -141,6 +139,7 @@
     "@icons-pack/react-simple-icons": "9.6.0",
     "@khmyznikov/pwa-install": "0.3.9",
     "@langchain/community": "^0.3.50",
+    "@lobechat/database": "workspace:*",
     "@lobechat/electron-client-ipc": "workspace:*",
     "@lobechat/electron-server-ipc": "workspace:*",
     "@lobechat/file-loaders": "workspace:*",

packages/database/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@lobechat/database",
+  "version": "1.0.0",
+  "private": true,
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "scripts": {
+    "test": "vitest",
+    "test:coverage": "vitest --coverage"
+  },
+  "dependencies": {
+    "@electric-sql/pglite": "^0.2.17",
+    "@lobechat/const": "workspace:*",
+    "@lobechat/types": "workspace:*",
+    "dayjs": "^1.11.13",
+    "drizzle-orm": "^0.44.4",
+    "nanoid": "^5.1.5",
+    "pg": "^8.16.3",
+    "random-words": "^2.0.1",
+    "ts-md5": "^2.0.1",
+    "ws": "^8.18.3"
+  }
+}

packages/database/src/client/db.ts

@@ -2,7 +2,6 @@ import { sql } from 'drizzle-orm';
 import { PgliteDatabase, drizzle } from 'drizzle-orm/pglite';
 import { Md5 } from 'ts-md5';
 
-import { DrizzleMigrationModel } from '@/database/models/drizzleMigration';
 import {
   ClientDBLoadingProgress,
   DatabaseLoadingState,
@@ -11,8 +10,9 @@ import {
 } from '@/types/clientDB';
 import { sleep } from '@/utils/sleep';
 
+import migrations from '../core/migrations.json';
+import { DrizzleMigrationModel } from '../models/drizzleMigration';
 import * as schema from '../schemas';
-import migrations from './migrations.json';
 
 const pgliteSchemaHashCache = 'LOBE_CHAT_PGLITE_SCHEMA_HASH';
 

packages/database/src/core/db-adaptor.ts

@@ -1,8 +1,8 @@
-import { isDesktop } from '@/const/version';
-import { getDBInstance } from '@/database/core/web-server';
-import { LobeChatDatabase } from '@/database/type';
+import { isDesktop } from '@lobechat/const';
 
+import { LobeChatDatabase } from '../type';
 import { getPgliteInstance } from './electron';
+import { getDBInstance } from './web-server';
 
 /**
  * 懒加载数据库实例

packages/database/src/core/dbForTest.ts

@@ -11,7 +11,7 @@ import { serverDBEnv } from '@/config/db';
 
 import * as schema from '../schemas';
 
-const migrationsFolder = join(__dirname, '../migrations');
+const migrationsFolder = join(__dirname, '../../migrations');
 
 export const getTestDBInstance = async () => {
   let connectionString = serverDBEnv.DATABASE_TEST_URL;

packages/database/src/core/electron.ts

@@ -4,13 +4,13 @@ import { drizzle as pgliteDrizzle } from 'drizzle-orm/pglite';
 import fs from 'node:fs';
 import { Md5 } from 'ts-md5';
 
-import { DrizzleMigrationModel } from '@/database/models/drizzleMigration';
-import * as schema from '@/database/schemas';
 import { electronIpcClient } from '@/server/modules/ElectronIPCClient';
 import { MigrationTableItem } from '@/types/clientDB';
 
-import migrations from '../client/migrations.json';
+import { DrizzleMigrationModel } from '../models/drizzleMigration';
+import * as schema from '../schemas';
 import { LobeChatDatabase } from '../type';
+import migrations from './migrations.json';
 
 // 用于实例管理的全局对象
 interface LobeGlobal {

packages/database/src/core/web-server.ts

@@ -5,13 +5,12 @@ import { Pool as NodePool } from 'pg';
 import ws from 'ws';
 
 import { serverDBEnv } from '@/config/db';
-import { isServerMode } from '@/const/version';
-import * as schema from '@/database/schemas';
 
+import * as schema from '../schemas';
 import { LobeChatDatabase } from '../type';
 
 export const getDBInstance = (): LobeChatDatabase => {
-  if (!isServerMode) return {} as any;
+  if (!(process.env.NEXT_PUBLIC_SERVICE_MODE === 'server')) return {} as any;
 
   if (!serverDBEnv.KEY_VAULTS_SECRET) {
     throw new Error(

packages/database/src/models/__tests__/_test_template.ts

@@ -2,9 +2,8 @@
 import { eq } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 
-import { LobeChatDatabase } from '@/database/type';
-
 import { sessionGroups, users } from '../../schemas';
+import { LobeChatDatabase } from '../../type';
 import { SessionGroupModel } from '../sessionGroup';
 import { getTestDB } from './_util';
 

packages/database/src/models/__tests__/_util.ts

@@ -1,11 +1,11 @@
-import { clientDB, initializeDB } from '@/database/client/db';
-import { LobeChatDatabase } from '@/database/type';
+import { clientDB, initializeDB } from '../../client/db';
+import { LobeChatDatabase } from '../../type';
 
 const isServerDBMode = process.env.TEST_SERVER_DB === '1';
 
 export const getTestDB = async () => {
   if (isServerDBMode) {
-    const { getTestDBInstance } = await import('@/database/core/dbForTest');
+    const { getTestDBInstance } = await import('../../core/dbForTest');
     return await getTestDBInstance();
   }
 

packages/database/src/models/__tests__/agent.test.ts

@@ -2,8 +2,7 @@
 import { eq } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 
-import { LobeChatDatabase } from '@/database/type';
-
+import { LobeChatDatabase } from '../../type';
 import {
   agents,
   agentsFiles,

packages/database/src/models/__tests__/aiModel.test.ts

@@ -2,10 +2,10 @@
 import { eq } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 
-import { LobeChatDatabase } from '@/database/type';
 import { AiProviderModelListItem } from '@/types/aiModel';
 
 import { AiModelSelectItem, NewAiModelItem, aiModels, users } from '../../schemas';
+import { LobeChatDatabase } from '../../type';
 import { AiModelModel } from '../aiModel';
 import { getTestDB } from './_util';
 

packages/database/src/models/__tests__/aiProvider.test.ts

@@ -3,10 +3,10 @@ import { ModelProvider } from '@lobechat/model-runtime';
 import { eq } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
-import { LobeChatDatabase } from '@/database/type';
 import { sleep } from '@/utils/sleep';
 
 import { aiProviders, users } from '../../schemas';
+import { LobeChatDatabase } from '../../type';
 import { AiProviderModel } from '../aiProvider';
 import { getTestDB } from './_util';
 

packages/database/src/models/__tests__/asyncTask.test.ts

@@ -2,10 +2,10 @@
 import { eq } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
-import { LobeChatDatabase } from '@/database/type';
 import { AsyncTaskStatus, AsyncTaskType } from '@/types/asyncTask';
 
 import { asyncTasks, users } from '../../schemas';
+import { LobeChatDatabase } from '../../type';
 import { ASYNC_TASK_TIMEOUT, AsyncTaskModel } from '../asyncTask';
 import { getTestDB } from './_util';
 

packages/database/src/models/__tests__/chunk.test.ts

@@ -2,8 +2,7 @@
 import { eq } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 
-import { LobeChatDatabase } from '@/database/type';
-import { uuid } from '@/utils/uuid';
+import { LobeChatDatabase } from '../../type';import { uuid } from '@/utils/uuid';
 
 import { chunks, embeddings, fileChunks, files, unstructuredChunks, users } from '../../schemas';
 import { ChunkModel } from '../chunk';

packages/database/src/models/__tests__/file.test.ts

@@ -2,10 +2,10 @@
 import { eq, inArray } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
-import { LobeChatDatabase } from '@/database/type';
 import { FilesTabs, SortType } from '@/types/files';
 
 import { files, globalFiles, knowledgeBaseFiles, knowledgeBases, users } from '../../schemas';
+import { LobeChatDatabase } from '../../type';
 import { FileModel } from '../file';
 import { getTestDB } from './_util';
 

packages/database/src/models/__tests__/generation.test.ts

@@ -2,8 +2,7 @@
 import { eq } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
-import { LobeChatDatabase } from '@/database/type';
-import { AsyncTaskStatus } from '@/types/asyncTask';
+import { LobeChatDatabase } from '../../type';import { AsyncTaskStatus } from '@/types/asyncTask';
 import { FileSource } from '@/types/files';
 import { ImageGenerationAsset } from '@/types/generation';
 

packages/database/src/models/__tests__/generationBatch.test.ts

@@ -2,8 +2,7 @@
 import { eq } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
-import { LobeChatDatabase } from '@/database/type';
-import { AsyncTaskStatus } from '@/types/asyncTask';
+import { LobeChatDatabase } from '../../type';import { AsyncTaskStatus } from '@/types/asyncTask';
 import { GenerationConfig } from '@/types/generation';
 
 import {

packages/database/src/models/__tests__/generationTopic.test.ts

@@ -2,8 +2,7 @@
 import { eq } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
-import { LobeChatDatabase } from '@/database/type';
-import { FileService } from '@/server/services/file';
+import { LobeChatDatabase } from '../../type';import { FileService } from '@/server/services/file';
 import { ImageGenerationTopic } from '@/types/generation';
 
 import { generationBatches, generationTopics, generations, users } from '../../schemas';

packages/database/src/models/__tests__/knowledgeBase.test.ts

@@ -2,8 +2,7 @@
 import { and, eq } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 
-import { LobeChatDatabase } from '@/database/type';
-import { sleep } from '@/utils/sleep';
+import { LobeChatDatabase } from '../../type';import { sleep } from '@/utils/sleep';
 
 import {
   NewKnowledgeBase,

packages/database/src/models/__tests__/message.test.ts

@@ -2,11 +2,10 @@ import dayjs from 'dayjs';
 import { eq } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
-import { getTestDB } from '@/database/models/__tests__/_util';
-import { LobeChatDatabase } from '@/database/type';
 import { MessageItem } from '@/types/message';
 import { uuid } from '@/utils/uuid';
 
+import { getTestDB } from '../../models/__tests__/_util';
 import {
   chunks,
   embeddings,
@@ -23,6 +22,7 @@ import {
   topics,
   users,
 } from '../../schemas';
+import { LobeChatDatabase } from '../../type';
 import { MessageModel } from '../message';
 import { codeEmbedding } from './fixtures/embedding';
 

packages/database/src/models/__tests__/plugin.test.ts

@@ -1,8 +1,7 @@
 // @vitest-environment node
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
-import { LobeChatDatabase } from '@/database/type';
-
+import { LobeChatDatabase } from '../../type';
 import { NewInstalledPlugin, userInstalledPlugins, users } from '../../schemas';
 import { PluginModel } from '../plugin';
 import { getTestDB } from './_util';

packages/database/src/models/__tests__/session.test.ts

@@ -2,8 +2,6 @@ import { and, eq, inArray } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
 import { DEFAULT_AGENT_CONFIG } from '@/const/settings';
-import { LobeChatDatabase } from '@/database/type';
-import { idGenerator } from '@/database/utils/idGenerator';
 
 import {
   NewSession,
@@ -16,6 +14,8 @@ import {
   topics,
   users,
 } from '../../schemas';
+import { LobeChatDatabase } from '../../type';
+import { idGenerator } from '../../utils/idGenerator';
 import { SessionModel } from '../session';
 import { getTestDB } from './_util';
 

packages/database/src/models/__tests__/sessionGroup.test.ts

@@ -2,9 +2,8 @@
 import { eq } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
-import { LobeChatDatabase } from '@/database/type';
-
 import { sessionGroups, users } from '../../schemas';
+import { LobeChatDatabase } from '../../type';
 import { SessionGroupModel } from '../sessionGroup';
 import { getTestDB } from './_util';
 

packages/database/src/models/__tests__/topic.test.ts

@@ -1,8 +1,7 @@
 import { eq, inArray } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
-import { LobeChatDatabase } from '@/database/type';
-
+import { LobeChatDatabase } from '../../type';
 import { messages, sessions, topics, users } from '../../schemas';
 import { CreateTopicParams, TopicModel } from '../topic';
 import { getTestDB } from './_util';

packages/database/src/models/_template.ts

@@ -1,8 +1,7 @@
 import { and, desc, eq } from 'drizzle-orm';
 
-import { LobeChatDatabase } from '@/database/type';
-
 import { NewSessionGroup, SessionGroupItem, sessionGroups } from '../schemas';
+import { LobeChatDatabase } from '../type';
 
 export class TemplateModel {
   private userId: string;

packages/database/src/models/agent.ts

@@ -7,8 +7,8 @@ import {
   agentsToSessions,
   files,
   knowledgeBases,
-} from '@/database/schemas';
-import { LobeChatDatabase } from '@/database/type';
+} from '../schemas';
+import { LobeChatDatabase } from '../type';
 
 export class AgentModel {
   private userId: string;

packages/database/src/models/aiModel.ts

@@ -1,6 +1,6 @@
 import { and, asc, desc, eq, inArray } from 'drizzle-orm';
 
-import { LobeChatDatabase } from '@/database/type';
+import { LobeChatDatabase } from '../type';
 import {
   AiModelSortMap,
   AiModelSourceEnum,