diff --git a/.env.example b/.env.example index 9f03b93e5a..ca394f4fdf 100644 --- a/.env.example +++ b/.env.example @@ -100,6 +100,8 @@ DOCREADER_TRANSPORT=grpc # ========== B1. 数据库 ⚠️ 必填 ========== # 主数据库类型：postgres / mysql / sqlite。 +# mysql 模式要求 MySQL 8.0.16+（CHECK 约束 / SKIP LOCKED / JSON_LENGTH / utf8mb4_0900_ai_ci）， +# 且 RETRIEVE_DRIVER 不得为 postgres 或 sqlite（向量检索需委托给外部引擎）。 DB_DRIVER=postgres # 数据库主机地址。 DB_HOST=postgres @@ -113,6 +115,23 @@ DB_PASSWORD=postgres123!@# DB_NAME=WeKnora # SQLite 驱动时使用（DB_DRIVER=sqlite），postgres/mysql 忽略。 # DB_PATH=./data/weknora.db +# +# MySQL 连接池与超时（仅 DB_DRIVER=mysql 时生效）： +# DB_CONNECT_TIMEOUT=10s # 连接超时 +# DB_READ_TIMEOUT=30s # 读超时 +# DB_WRITE_TIMEOUT=30s # 写超时 +# DB_MAX_OPEN_CONNS=50 # 最大打开连接数 +# DB_MAX_IDLE_CONNS=10 # 最大空闲连接数 +# DB_CONN_MAX_LIFETIME=10m # 连接最大存活时间 +# DB_CONN_MAX_IDLE_TIME=5m # 空闲连接最大存活时间 +# +# MySQL TLS（外部或要求加密传输的 MySQL；仅 DB_DRIVER=mysql 时生效）： +# DB_USE_TLS=false +# DB_TLS_SERVER_NAME= # SNI / 证书校验名称 +# DB_TLS_CA= # CA PEM 文件路径 +# DB_TLS_CERT= # mTLS 客户端证书 PEM（必须与 DB_TLS_KEY 同时设置） +# DB_TLS_KEY= # mTLS 客户端私钥 PEM +# DB_TLS_INSECURE_SKIP_VERIFY=false # 不安全，仅用于本地自签证书 # ========== B2. Redis / 流处理 / Asynq 队列 ========== # --- 流处理后端与 Redis 连接 --- @@ -236,6 +255,7 @@ LOCAL_STORAGE_BASE_DIR=/data/files # ========== C1. 向量库 / 检索引擎 ========== # 向量存储类型（逗号分隔可多驱动）：postgres / elasticsearch_v7 / elasticsearch_v8 / # opensearch / qdrant / milvus / weaviate / doris / tencent_vectordb。 +# 注意：DB_DRIVER=mysql 时，此处不能填 postgres（启动会校验失败）。 RETRIEVE_DRIVER=postgres # 多向量库并行检索超时（秒，RETRIEVE_DRIVER 含多个驱动时生效）。 # MULTI_STORE_RETRIEVE_TIMEOUT_SEC= diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml index 1f7bce20da..8c25592835 100644 --- a/.github/workflows/app.yml +++ b/.github/workflows/app.yml @@ -96,3 +96,41 @@ jobs: - name: Build server run: go build ./cmd/server + + # The repository-layer tests in the job above run on SQLite, which silently + # accepts SQL that MySQL rejects — the PostgreSQL bare-key `col->>'key'` form + # is the clearest example. Without a real server in CI, the MySQL integration + # tests skip themselves and stop being a signal. + mysql-integration: + name: MySQL integration tests + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: weknora-test + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -pweknora-test" + --health-interval=5s + --health-timeout=5s + --health-retries=20 + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-go@v6 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - name: Download modules + run: go mod download + + # Each test creates and drops its own database, so it only needs a server + # to connect to; `mysql` is the always-present bootstrap schema. + - name: Run MySQL integration tests + env: + WEKNORA_MYSQL_TEST_DSN: "root:weknora-test@tcp(127.0.0.1:3306)/mysql?charset=utf8mb4&collation=utf8mb4_0900_ai_ci&parseTime=true&loc=UTC" + run: go test ./internal/application/repository/ -run TestMySQL -v diff --git a/docker-compose.mysql.yml b/docker-compose.mysql.yml new file mode 100644 index 0000000000..50dc0916de --- /dev/null +++ b/docker-compose.mysql.yml @@ -0,0 +1,88 @@ +# MySQL override for docker-compose.yml +# +# Swap the metadata database from PostgreSQL to MySQL 8.0+. Usage: +# +# docker compose --profile qdrant -f docker-compose.yml -f docker-compose.mysql.yml up -d +# +# Required environment (in .env or shell): +# DB_DRIVER=mysql +# DB_HOST=postgres # the service name is unchanged so app's depends_on resolves +# DB_PORT=3306 +# DB_USER=weknora +# DB_PASSWORD= +# DB_NAME=weknora +# MYSQL_ROOT_PASSWORD= # required; no fixed fallback +# RETRIEVE_DRIVER=qdrant # MUST NOT be postgres or sqlite — MySQL mode +# # never creates the embeddings table. Use an +# # external engine: qdrant, milvus, elasticsearch_v8, +# # opensearch, doris, weaviate, tencent_vectordb. +# +# The app validates this combination at startup and fails fast with an +# actionable message if RETRIEVE_DRIVER is misconfigured. +# +# Limitations under MySQL: +# - Wiki keyword search uses multi-column LIKE instead of PostgreSQL's +# to_tsvector / pg_trgm. Matching and ranking semantics differ. +# - The full / langfuse profile assumes the `postgres` service is really +# PostgreSQL; this override replaces it with MySQL, so Langfuse init +# will fail. Deploy a separate PostgreSQL instance for Langfuse. +# +# NOTE: this override only changes the metadata database. To actually run +# retrieval you also need the chosen external engine (qdrant/milvus/...) +# declared as a service and reachable. See docs/使用其他向量数据库.md. + +services: + postgres: # keep the name so the app's depends_on: postgres resolves + image: mysql:8.0.37 # pinned supported patch version + container_name: WeKnora-mysql + environment: + # Clear the inherited PostgreSQL env vars — without this the MySQL + # image would still receive POSTGRES_USER / POSTGRES_PASSWORD / + # POSTGRES_DB (harmless but confusing) from the base compose file. + - POSTGRES_USER= + - POSTGRES_PASSWORD= + - POSTGRES_DB= + - MYSQL_USER=${DB_USER:?DB_USER is required for MySQL deployments} + - MYSQL_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required for MySQL deployments} + - MYSQL_DATABASE=${DB_NAME:?DB_NAME is required for MySQL deployments} + # Root password is deliberately separate from DB_PASSWORD so the app + # credential and the admin credential can be rotated independently. + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:?MYSQL_ROOT_PASSWORD is required for MySQL deployments} + # The base docker-compose.yml defines this service's `volumes` as a LIST + # (`- postgres-data:/var/lib/postgresql/data`). Docker Compose normally + # *appends* list entries across files, so a plain `- mysql-data:...` + # override would leave both volumes mounted (postgres-data dangling and + # unused). Using the `!override` tag tells Compose to REPLACE the list + # entirely, so only mysql-data is mounted. + # + # Requires Docker Compose v2.24+ (Jan 2024) for the `!override` tag. + # On older Compose, drop `!override` and accept the harmless extra + # empty postgres-data mount at /var/lib/postgresql/data. + volumes: !override + - mysql-data:/var/lib/mysql + networks: + - WeKnora-network + healthcheck: + # mysqladmin ping returns success once the server answers, even before + # accounts are fully provisioned. No password is needed for a basic + # connectivity check, so we intentionally omit -u/-p (which would also + # leak credentials into the container's inspect output). + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + interval: 10s + timeout: 10s + retries: 5 + start_period: 30s + restart: unless-stopped + stop_grace_period: 1m + # utf8mb4 is the default in MySQL 8.0, but set it explicitly so the + # character set is deterministic regardless of the server's config file. + # --default-authentication-plugin=caching_sha2_password is intentionally + # NOT set: it was deprecated in 8.0.34 and removed in 8.4. caching_sha2 + # is already the default in 8.0. + command: + - --character-set-server=utf8mb4 + - --collation-server=utf8mb4_0900_ai_ci + - --default-time-zone=+00:00 + +volumes: + mysql-data: diff --git a/docker/Dockerfile.app b/docker/Dockerfile.app index 38e12b06c4..ce2310fba7 100644 --- a/docker/Dockerfile.app +++ b/docker/Dockerfile.app @@ -21,8 +21,9 @@ RUN if [ -n "$APK_MIRROR_ARG" ]; then \ apt-get update && \ apt-get install -y git build-essential libsqlite3-dev -# Install migrate tool -RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest +# Install the migration CLI with both metadata database drivers. Keep the +# version aligned with go.mod so local and container behavior stay identical. +RUN go install -tags 'postgres mysql' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.19.1 # Copy go mod and sum files COPY go.mod go.sum ./ diff --git a/docs/mysql-primary-database.md b/docs/mysql-primary-database.md new file mode 100644 index 0000000000..68cbde3d48 --- /dev/null +++ b/docs/mysql-primary-database.md @@ -0,0 +1,239 @@ +# MySQL 作为主数据库 + +WeKnora 支持 MySQL 8.0.16+ 作为元数据数据库（PostgreSQL 之外的选项）。 + +## 最低要求 + +- **MySQL 8.0.16+**（CHECK 约束从 8.0.16 开始强制执行；更早的 8.0.x 版本会静默忽略 CHECK） +- 也支持 8.4.x 和 9.x +- **不支持 MariaDB**（JSON / SKIP LOCKED / CHECK / utf8mb4_0900_ai_ci 语义与 MySQL 8 不兼容） +- 字符集：`utf8mb4`（支持 4 字节 UTF-8：emoji、CJK 扩展） +- 排序规则：`utf8mb4_0900_ai_ci`（大小写不敏感、重音不敏感） + +## Metadata 与 Retriever 的关系 + +MySQL 只接管**元数据层**（工作空间、知识库、文档、消息、Wiki 等）。**向量检索必须委托给外部引擎**： + +``` +DB_DRIVER=mysql → 元数据存储在 MySQL +RETRIEVE_DRIVER=qdrant → 向量索引存储在 Qdrant（或其他外部引擎） +``` + +启动时会校验组合：`DB_DRIVER=mysql` + `RETRIEVE_DRIVER=postgres` 或 `sqlite` 会被拒绝（因为 postgres/sqlite retriever 依赖 MySQL 模式下不存在的 embeddings 表）。 + +合法的外部引擎：`qdrant`、`milvus`、`elasticsearch_v8`、`weaviate`、`doris`、`tencent_vectordb`。 + +## 快速开始（Docker Compose） + +```bash +# 1. 设置环境变量 +# 在 .env 中： +DB_DRIVER=mysql +DB_HOST=postgres # 服务名不变，docker-compose override 会将其替换为 MySQL +DB_PORT=3306 +DB_USER=weknora +DB_PASSWORD= +DB_NAME=weknora +MYSQL_ROOT_PASSWORD= +RETRIEVE_DRIVER=qdrant +QDRANT_HOST=qdrant +QDRANT_PORT=6334 + +# 2. 启动（需要 qdrant profile 来运行向量引擎） +docker compose --profile qdrant -f docker-compose.yml -f docker-compose.mysql.yml up -d +``` + +## 迁移管理 + +WeKnora 使用 [golang-migrate](https://github.com/golang-migrate/migrate) 管理数据库 schema。 + +### MySQL 迁移文件 + +MySQL 使用独立的 squash baseline： + +- `migrations/mysql/000000_init.up.sql` 创建完整 schema。 +- `migrations/mysql/000000_init.down.sql` 回滚完整 schema。 + +MySQL schema 永久只维护这两个 `000000` baseline 文件。后续 schema 变化也必须直接同步到这两个文件， +不得新增 MySQL 增量 migration。该模式只面向全新 MySQL 部署，不提供已部署 MySQL 实例的增量升级链路； +不要把 PostgreSQL migration 文件直接用于 MySQL。 + +### 手动迁移 + +```bash +# 使用迁移脚本 +DB_DRIVER=mysql ./scripts/migrate.sh up +DB_DRIVER=mysql ./scripts/migrate.sh down +DB_DRIVER=mysql ./scripts/migrate.sh version +DB_DRIVER=mysql ./scripts/migrate.sh force +``` + +手动脚本继续使用官方 `migrate` CLI。该 CLI 的自定义 MySQL TLS 配置不能设置独立于连接地址的 SNI， +因此 `DB_TLS_SERVER_NAME` 与 `DB_HOST` 不一致时脚本会拒绝执行；请改用证书对应的 DNS 名作为连接地址， +或让应用通过 `AUTO_MIGRATE` 执行迁移。手动 mTLS 迁移还必须同时提供 `DB_TLS_CA`。 + +### Dirty 状态处理 + +MySQL 的 DDL 语句（CREATE TABLE、ALTER TABLE 等）会隐式提交事务，因此迁移失败后无法回滚到一致状态。 + +**MySQL 模式下，dirty 迁移状态不会自动恢复**（PostgreSQL/SQLite 会自动 Force + 重试）。如果迁移失败： + +1. 检查已创建的表（可能只创建了部分 schema） +2. 决定是修复还是重建数据库 +3. 手动执行 `./scripts/migrate.sh force ` 或删除数据库后重新迁移 + +### Rollback + +```bash +DB_DRIVER=mysql ./scripts/migrate.sh down +# 验证：down 后应剩 0 张业务表 +``` + +## 连接配置 + +### DSN 参数 + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `DB_HOST` | （必填） | MySQL 主机地址（支持 IPv6） | +| `DB_PORT` | 3306 | MySQL 端口 | +| `DB_USER` | （必填） | 数据库用户名 | +| `DB_PASSWORD` | （必填） | 数据库密码 | +| `DB_NAME` | （必填） | 数据库名称 | + +### 连接池 + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `DB_MAX_OPEN_CONNS` | 50 | 最大打开连接数 | +| `DB_MAX_IDLE_CONNS` | 10 | 最大空闲连接数 | +| `DB_CONN_MAX_LIFETIME` | 10m | 连接最大存活时间 | +| `DB_CONN_MAX_IDLE_TIME` | 5m | 空闲连接最大存活时间 | + +### 超时 + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `DB_CONNECT_TIMEOUT` | 10s | 连接超时 | +| `DB_READ_TIMEOUT` | 30s | 读超时 | +| `DB_WRITE_TIMEOUT` | 30s | 写超时 | + +### TLS + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `DB_USE_TLS` | false | 启用 TLS | +| `DB_TLS_SERVER_NAME` | | TLS 服务器名（SNI） | +| `DB_TLS_CA` | | CA 证书文件路径 | +| `DB_TLS_CERT` | | 客户端证书文件路径 | +| `DB_TLS_KEY` | | 客户端私钥文件路径 | +| `DB_TLS_INSECURE_SKIP_VERIFY` | false | 跳过证书验证（仅开发环境） | + +### 时区 + +所有时间戳以 UTC 存储： +- 连接 `loc=UTC` +- 每条应用和迁移连接都设置 session `time_zone='+00:00'` +- 服务器仍建议设置 `--default-time-zone=+00:00` +- 迁移 baseline 开头 `SET time_zone = '+00:00'` +- 所有时间列使用 `DATETIME(6)`（微秒精度） + +## Helm 部署 + +MySQL 和 PostgreSQL 内部部署互斥。启用 MySQL 时**必须**同时关闭 PostgreSQL： + +```bash +# --set 方式（注意必须同时设置 postgresql.enabled=false） +helm install weknora ./helm \ + --set mysql.enabled=true \ + --set postgresql.enabled=false \ + --set qdrant.enabled=true \ + --set app.env.RETRIEVE_DRIVER=qdrant \ + --set secrets.redisPassword=... \ + --set secrets.jwtSecret=... +``` + +```yaml +# values.yaml 方式 +mysql: + enabled: true + auth: + database: WeKnora + username: weknora + # 生产环境建议使用已有 Secret，包含 MYSQL_DATABASE、MYSQL_USER、 + # MYSQL_PASSWORD、MYSQL_ROOT_PASSWORD。 + existingSecret: weknora-mysql-credentials + +postgresql: + enabled: false # 必须显式关闭 + +app: + env: + RETRIEVE_DRIVER: qdrant # MySQL 模式必须使用外部检索引擎 + +qdrant: + enabled: true + persistence: + size: 20Gi +``` + +使用外部 MySQL 与外部 Qdrant 时，不渲染内部数据库： + +```yaml +postgresql: + enabled: false + +database: + external: true + driver: mysql + host: "your-mysql.internal" + port: 3306 + existingSecret: external-mysql-credentials # DB_USER / DB_PASSWORD / DB_NAME + tls: + enabled: true + serverName: "your-mysql.internal" + existingSecret: external-mysql-tls + caFile: ca.crt + # mTLS 可同时设置 certFile: tls.crt 与 keyFile: tls.key + +app: + env: + RETRIEVE_DRIVER: qdrant + +qdrant: + enabled: false + connection: + host: "your-qdrant.internal" + port: 6334 + collection: weknora_embeddings + useTLS: true + auth: + existingSecret: external-qdrant-credentials + apiKeyKey: QDRANT_API_KEY +``` + +## Wiki 搜索差异 + +MySQL 和 PostgreSQL 在 Wiki 搜索功能上存在已知差异： + +| 功能 | PostgreSQL | MySQL | +|------|-----------|-------| +| 全文搜索 | `to_tsvector` + GIN 索引（词级匹配） | `LIKE '%query%'`（子串匹配） | +| 相似页面 | `pg_trgm` 相似度 + GIN 索引 | `LIKE` 近似匹配（粗粒度） | +| source_refs 查询 | `@>` + GIN 索引 | `JSON_CONTAINS`（KB 分区内扫描） | + +两种数据库的匹配和排序语义不同，结果集合不可直接比较。MySQL 的 `LIKE` 与 +`JSON_CONTAINS` 查询在大型知识库中也可能更慢。 + +## Schema 上的两处刻意差异 + +以下两项是 MySQL schema 与 PostgreSQL 的有意分歧，运维时可能会注意到： + +- **`knowledges.metadata_external_id` 生成列**。PostgreSQL 用表达式索引 + `(knowledge_base_id, (metadata->>'external_id'))` 支撑数据源同步时的按外部 ID + 查询。MySQL 没有表达式索引，改用一个虚拟生成列加前缀索引来提供同一条查询路径。 + 它由 `metadata` 自动派生，不需要也不应该被写入。 + +- **`chunks.seq_id` / `knowledge_tags.seq_id` 的自增起始值**分别是 `100000000` + 与 `10000000`，与 PostgreSQL 序列的起始值一致。FAQ 导入允许调用方指定小于起始值 + 的 `seq_id`，因此起始值以下是导入专用的保留区间；自动生成的值不能进入该区间。 diff --git a/go.mod b/go.mod index 35593daa61..9433fc25cc 100644 --- a/go.mod +++ b/go.mod @@ -84,6 +84,7 @@ require ( google.golang.org/protobuf v1.36.11 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 + gorm.io/driver/mysql v1.6.0 gorm.io/driver/postgres v1.6.0 gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.31.1 diff --git a/go.sum b/go.sum index e9e2894277..383ce98660 100644 --- a/go.sum +++ b/go.sum @@ -3961,6 +3961,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg= +gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo= gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= diff --git a/helm/README.md b/helm/README.md index e685e49c94..893f06792e 100644 --- a/helm/README.md +++ b/helm/README.md @@ -190,6 +190,25 @@ helm install weknora ./helm \ | `postgresql.persistence.enabled` | Enable persistence | `true` | | `postgresql.persistence.size` | PVC size | `10Gi` | +### MySQL and External Database + +MySQL is a metadata database and requires an external retriever such as +Qdrant. Configure only one of `postgresql.enabled`, `mysql.enabled`, and +`database.external`. + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `mysql.enabled` | Enable in-chart MySQL | `false` | +| `mysql.auth.existingSecret` | Secret with `MYSQL_DATABASE`, `MYSQL_USER`, `MYSQL_PASSWORD`, `MYSQL_ROOT_PASSWORD` | `""` | +| `mysql.primary.persistence.size` | MySQL PVC size | `20Gi` | +| `database.external` | Use an externally managed database | `false` | +| `database.driver` | External driver (`postgres` or `mysql`) | `postgres` | +| `database.host` | External database hostname | `""` | +| `database.port` | External port; empty selects the driver default | `""` | +| `database.existingSecret` | Optional dedicated Secret with `DB_USER`, `DB_PASSWORD`, `DB_NAME`; otherwise reuse the application Secret | `""` | +| `database.tls.enabled` | Require TLS for external MySQL | `false` | +| `database.tls.existingSecret` | Secret containing CA/client certificate files | `""` | + ### Redis | Parameter | Description | Default | @@ -215,11 +234,11 @@ helm install weknora ./helm \ | Parameter | Description | Default | |-----------|-------------|---------| | `secrets.dbUser` | Database username | `postgres` | -| `secrets.dbPassword` | Database password | `""` (required) | +| `secrets.dbPassword` | In-chart PostgreSQL password | `""` (required with PostgreSQL) | | `secrets.dbName` | Database name | `weknora` | | `secrets.redisPassword` | Redis password | `""` (required) | | `secrets.jwtSecret` | JWT signing secret | `""` (required) | -| `secrets.existingSecret` | Use existing secret | `""` | +| `secrets.existingSecret` | Existing application Secret; also supplies `DB_*` for PostgreSQL or an external database without `database.existingSecret` | `""` | ### Optional Components @@ -229,7 +248,11 @@ These map to docker-compose profiles: |-----------|-------------|---------| | `minio.enabled` | Enable MinIO storage | `false` | | `neo4j.enabled` | Enable Neo4j (GraphRAG) | `false` | -| `qdrant.enabled` | Enable Qdrant vector DB | `false` | +| `qdrant.enabled` | Enable in-chart Qdrant vector DB | `false` | +| `qdrant.connection.host` | External Qdrant host; defaults to `qdrant` when enabled | `""` | +| `qdrant.connection.port` | Qdrant gRPC port | `6334` | +| `qdrant.auth.existingSecret` | Secret containing the Qdrant API key | `""` | +| `qdrant.persistence.size` | Qdrant PVC size | `10Gi` | ## Security Best Practices diff --git a/helm/templates/NOTES.txt b/helm/templates/NOTES.txt index d74b49468c..630502967d 100644 --- a/helm/templates/NOTES.txt +++ b/helm/templates/NOTES.txt @@ -58,7 +58,11 @@ Components deployed: {{- if .Values.docreader.enabled }} - Document Reader ({{ include "weknora.docreader.image" . }}) {{- end }} -{{- if .Values.postgresql.enabled }} +{{- if .Values.mysql.enabled }} + - MySQL ({{ include "weknora.mysql.image" . }}) -- DB_DRIVER=mysql +{{- else if .Values.database.external }} + - External database (driver={{ .Values.database.driver }}, host={{ .Values.database.host }}) +{{- else if .Values.postgresql.enabled }} - PostgreSQL/ParadeDB ({{ include "weknora.postgresql.image" . }}) {{- end }} {{- if .Values.redis.enabled }} diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index c8e75363c5..2941a9e0be 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -184,6 +184,32 @@ securityContext: {{- end }} {{- end }} +{{/* +Return the MySQL image string (values.mysql.image is a full repo:tag). +*/}} +{{- define "weknora.mysql.image" -}} +{{- .Values.mysql.image -}} +{{- end }} + +{{/* +Return the MySQL credential Secret name. An operator-managed Secret must +contain MYSQL_DATABASE, MYSQL_USER, MYSQL_PASSWORD, and MYSQL_ROOT_PASSWORD. +*/}} +{{- define "weknora.mysqlSecretName" -}} +{{- if .Values.mysql.auth.existingSecret }} +{{- .Values.mysql.auth.existingSecret }} +{{- else }} +{{- printf "%s-mysql" (include "weknora.fullname" .) }} +{{- end }} +{{- end }} + +{{/* +Return the Qdrant image with tag. +*/}} +{{- define "weknora.qdrant.image" -}} +{{- printf "%s:%s" .Values.qdrant.image.repository .Values.qdrant.image.tag }} +{{- end }} + {{/* Container security context. */}} diff --git a/helm/templates/app.yaml b/helm/templates/app.yaml index f0e18a22ff..1576ff8c37 100644 --- a/helm/templates/app.yaml +++ b/helm/templates/app.yaml @@ -52,6 +52,71 @@ spec: - name: TZ value: {{ .Values.app.env.TZ | quote }} # Database configuration + {{- if .Values.mysql.enabled }} + - name: DB_DRIVER + value: "mysql" + - name: DB_HOST + value: "mysql" + - name: DB_PORT + value: "3306" + - name: DB_USER + valueFrom: + secretKeyRef: + name: {{ include "weknora.mysqlSecretName" . }} + key: MYSQL_USER + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "weknora.mysqlSecretName" . }} + key: MYSQL_PASSWORD + - name: DB_NAME + valueFrom: + secretKeyRef: + name: {{ include "weknora.mysqlSecretName" . }} + key: MYSQL_DATABASE + {{- else if .Values.database.external }} + {{- $databaseSecretName := default (include "weknora.secretName" .) .Values.database.existingSecret }} + - name: DB_DRIVER + value: {{ .Values.database.driver | quote }} + - name: DB_HOST + value: {{ .Values.database.host | quote }} + - name: DB_PORT + value: {{ .Values.database.port | default (ternary 3306 5432 (eq (lower .Values.database.driver) "mysql")) | quote }} + - name: DB_USER + valueFrom: + secretKeyRef: + name: {{ $databaseSecretName }} + key: DB_USER + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $databaseSecretName }} + key: DB_PASSWORD + - name: DB_NAME + valueFrom: + secretKeyRef: + name: {{ $databaseSecretName }} + key: DB_NAME + optional: true + {{- if and (eq (lower .Values.database.driver) "mysql") .Values.database.tls.enabled }} + - name: DB_USE_TLS + value: "true" + - name: DB_TLS_SERVER_NAME + value: {{ .Values.database.tls.serverName | quote }} + - name: DB_TLS_INSECURE_SKIP_VERIFY + value: {{ .Values.database.tls.insecureSkipVerify | quote }} + {{- if .Values.database.tls.caFile }} + - name: DB_TLS_CA + value: {{ printf "/etc/weknora/mysql-tls/%s" .Values.database.tls.caFile | quote }} + {{- end }} + {{- if .Values.database.tls.certFile }} + - name: DB_TLS_CERT + value: {{ printf "/etc/weknora/mysql-tls/%s" .Values.database.tls.certFile | quote }} + - name: DB_TLS_KEY + value: {{ printf "/etc/weknora/mysql-tls/%s" .Values.database.tls.keyFile | quote }} + {{- end }} + {{- end }} + {{- else }} - name: DB_DRIVER value: "postgres" - name: DB_HOST @@ -73,6 +138,7 @@ spec: secretKeyRef: name: {{ include "weknora.secretName" . }} key: DB_NAME + {{- end }} # Redis configuration - name: REDIS_ADDR value: "redis:6379" @@ -107,6 +173,29 @@ spec: # Retrieval & Storage - name: RETRIEVE_DRIVER value: {{ .Values.app.env.RETRIEVE_DRIVER | quote }} + {{- $usesQdrant := false }} + {{- range (splitList "," (.Values.app.env.RETRIEVE_DRIVER | default "")) }} + {{- if eq (trim .) "qdrant" }} + {{- $usesQdrant = true }} + {{- end }} + {{- end }} + {{- if $usesQdrant }} + - name: QDRANT_HOST + value: {{ .Values.qdrant.connection.host | default "qdrant" | quote }} + - name: QDRANT_PORT + value: {{ .Values.qdrant.connection.port | quote }} + - name: QDRANT_COLLECTION + value: {{ .Values.qdrant.connection.collection | quote }} + - name: QDRANT_USE_TLS + value: {{ .Values.qdrant.connection.useTLS | quote }} + {{- if .Values.qdrant.auth.existingSecret }} + - name: QDRANT_API_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.qdrant.auth.existingSecret }} + key: {{ .Values.qdrant.auth.apiKeyKey }} + {{- end }} + {{- end }} - name: STORAGE_TYPE value: {{ .Values.app.env.STORAGE_TYPE | quote }} - name: LOCAL_STORAGE_BASE_DIR @@ -145,6 +234,11 @@ spec: volumeMounts: - name: data-files mountPath: /data/files + {{- if and .Values.database.external (eq (lower .Values.database.driver) "mysql") .Values.database.tls.enabled .Values.database.tls.existingSecret }} + - name: mysql-tls + mountPath: /etc/weknora/mysql-tls + readOnly: true + {{- end }} resources: {{- toYaml .Values.app.resources | nindent 12 }} {{- with .Values.app.livenessProbe }} @@ -163,6 +257,11 @@ spec: {{- else }} emptyDir: {} {{- end }} + {{- if and .Values.database.external (eq (lower .Values.database.driver) "mysql") .Values.database.tls.enabled .Values.database.tls.existingSecret }} + - name: mysql-tls + secret: + secretName: {{ .Values.database.tls.existingSecret }} + {{- end }} {{- with .Values.app.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/templates/mysql.yaml b/helm/templates/mysql.yaml new file mode 100644 index 0000000000..c8056e9746 --- /dev/null +++ b/helm/templates/mysql.yaml @@ -0,0 +1,198 @@ +{{/* +Copyright 2025 Tencent +SPDX-License-Identifier: MIT + +MySQL Deployment, Service, PVC, and Secret. +Rendered only when mysql.enabled is true. +*/}} +{{- if .Values.mysql.enabled }} +{{/* Reuse previously generated random passwords on upgrade. */}} +{{- $secretName := include "weknora.mysqlSecretName" . }} +{{- $managedSecretName := printf "%s-mysql" (include "weknora.fullname" .) }} +{{- $existing := dict }} +{{- if not .Values.mysql.auth.existingSecret }} +{{- $existing = lookup "v1" "Secret" .Release.Namespace $managedSecretName }} +{{- end }} +{{- $existingPassword := "" }} +{{- $existingRootPassword := "" }} +{{- if and $existing $existing.data }} + {{- if index $existing.data "MYSQL_PASSWORD" }} + {{- $existingPassword = index $existing.data "MYSQL_PASSWORD" | b64dec }} + {{- end }} + {{- if index $existing.data "MYSQL_ROOT_PASSWORD" }} + {{- $existingRootPassword = index $existing.data "MYSQL_ROOT_PASSWORD" | b64dec }} + {{- end }} +{{- end }} +{{- $mysqlPassword := .Values.mysql.auth.password | default $existingPassword | default (randAlphaNum 24) }} +{{- $mysqlRootPassword := .Values.mysql.auth.rootPassword | default $existingRootPassword | default (randAlphaNum 24) }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "weknora.fullname" . }}-mysql + namespace: {{ .Release.Namespace }} + labels: + {{- include "weknora.componentLabels" (dict "component" "database" "context" .) | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "weknora.componentSelectorLabels" (dict "component" "database" "context" .) | nindent 6 }} + # Use Recreate strategy for database to avoid data corruption + strategy: + type: Recreate + template: + metadata: + labels: + {{- include "weknora.componentSelectorLabels" (dict "component" "database" "context" .) | nindent 8 }} + spec: + {{- include "weknora.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "weknora.serviceAccountName" . }} + {{- with .Values.global.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: mysql + image: {{ .Values.mysql.image }} + imagePullPolicy: IfNotPresent + {{- with .Values.mysql.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + args: + - --character-set-server=utf8mb4 + - --collation-server=utf8mb4_0900_ai_ci + - --default-time-zone=+00:00 + ports: + - containerPort: 3306 + name: mysql + protocol: TCP + env: + - name: MYSQL_DATABASE + valueFrom: + secretKeyRef: + name: {{ $secretName }} + key: MYSQL_DATABASE + - name: MYSQL_USER + valueFrom: + secretKeyRef: + name: {{ $secretName }} + key: MYSQL_USER + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $secretName }} + key: MYSQL_PASSWORD + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $secretName }} + key: MYSQL_ROOT_PASSWORD + volumeMounts: + - name: mysql-data + mountPath: /var/lib/mysql + resources: + {{- toYaml .Values.mysql.resources | nindent 12 }} + livenessProbe: + exec: + command: + - sh + - -c + - mysqladmin ping -h localhost + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + readinessProbe: + exec: + command: + - sh + - -c + - mysqladmin ping -h localhost + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + volumes: + - name: mysql-data + {{- if .Values.mysql.primary.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ .Values.mysql.primary.persistence.existingClaim | default (printf "%s-mysql" (include "weknora.fullname" .)) }} + {{- else }} + emptyDir: {} + {{- end }} + {{- with .Values.mysql.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.mysql.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.mysql.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + # Service name is "mysql" - app references this as DB_HOST when mysql.enabled + name: mysql + namespace: {{ .Release.Namespace }} + labels: + {{- include "weknora.componentLabels" (dict "component" "database" "context" .) | nindent 4 }} +spec: + type: ClusterIP + selector: + {{- include "weknora.componentSelectorLabels" (dict "component" "database" "context" .) | nindent 4 }} + ports: + - name: mysql + port: 3306 + targetPort: mysql + protocol: TCP +--- +{{/* MySQL PVC */}} +{{- if and .Values.mysql.primary.persistence.enabled (not .Values.mysql.primary.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "weknora.fullname" . }}-mysql + namespace: {{ .Release.Namespace }} + labels: + {{- include "weknora.componentLabels" (dict "component" "database" "context" .) | nindent 4 }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.mysql.primary.persistence.size }} + {{- $storageClass := .Values.mysql.primary.persistence.storageClass }} + {{- if $storageClass }} + {{- if eq $storageClass "-" }} + storageClassName: "" + {{- else }} + storageClassName: {{ $storageClass | quote }} + {{- end }} + {{- else }} + {{- include "weknora.storageClass" . | nindent 2 }} + {{- end }} +--- +{{- end }} +{{- if not .Values.mysql.auth.existingSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "weknora.labels" . | nindent 4 }} +type: Opaque +stringData: + MYSQL_DATABASE: {{ .Values.mysql.auth.database | quote }} + MYSQL_USER: {{ .Values.mysql.auth.username | quote }} + MYSQL_PASSWORD: {{ $mysqlPassword | quote }} + MYSQL_ROOT_PASSWORD: {{ $mysqlRootPassword | quote }} +{{- end }} +{{- end }} diff --git a/helm/templates/qdrant.yaml b/helm/templates/qdrant.yaml new file mode 100644 index 0000000000..9a8c23b919 --- /dev/null +++ b/helm/templates/qdrant.yaml @@ -0,0 +1,139 @@ +{{/* +Qdrant Deployment, Service, and optional PVC. +*/}} +{{- if .Values.qdrant.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "weknora.fullname" . }}-qdrant + namespace: {{ .Release.Namespace }} + labels: + {{- include "weknora.componentLabels" (dict "component" "retriever" "context" .) | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "weknora.componentSelectorLabels" (dict "component" "retriever" "context" .) | nindent 6 }} + strategy: + type: Recreate + template: + metadata: + labels: + {{- include "weknora.componentSelectorLabels" (dict "component" "retriever" "context" .) | nindent 8 }} + spec: + {{- include "weknora.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ include "weknora.serviceAccountName" . }} + {{- with .Values.global.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: qdrant + image: {{ include "weknora.qdrant.image" . }} + imagePullPolicy: {{ .Values.qdrant.image.pullPolicy }} + {{- with .Values.qdrant.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 6333 + protocol: TCP + - name: grpc + containerPort: 6334 + protocol: TCP + {{- if .Values.qdrant.auth.existingSecret }} + env: + - name: QDRANT__SERVICE__API_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.qdrant.auth.existingSecret }} + key: {{ .Values.qdrant.auth.apiKeyKey }} + {{- end }} + volumeMounts: + - name: qdrant-data + mountPath: /qdrant/storage + resources: + {{- toYaml .Values.qdrant.resources | nindent 12 }} + livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 6 + readinessProbe: + tcpSocket: + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 6 + volumes: + - name: qdrant-data + {{- if .Values.qdrant.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ .Values.qdrant.persistence.existingClaim | default (printf "%s-qdrant" (include "weknora.fullname" .)) }} + {{- else }} + emptyDir: {} + {{- end }} + {{- with .Values.qdrant.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.qdrant.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.qdrant.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: qdrant + namespace: {{ .Release.Namespace }} + labels: + {{- include "weknora.componentLabels" (dict "component" "retriever" "context" .) | nindent 4 }} +spec: + type: {{ .Values.qdrant.service.type }} + selector: + {{- include "weknora.componentSelectorLabels" (dict "component" "retriever" "context" .) | nindent 4 }} + ports: + - name: http + port: {{ .Values.qdrant.service.httpPort }} + targetPort: http + protocol: TCP + - name: grpc + port: {{ .Values.qdrant.service.grpcPort }} + targetPort: grpc + protocol: TCP +{{- if and .Values.qdrant.persistence.enabled (not .Values.qdrant.persistence.existingClaim) }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "weknora.fullname" . }}-qdrant + namespace: {{ .Release.Namespace }} + labels: + {{- include "weknora.componentLabels" (dict "component" "retriever" "context" .) | nindent 4 }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.qdrant.persistence.size }} + {{- $storageClass := .Values.qdrant.persistence.storageClass }} + {{- if $storageClass }} + {{- if eq $storageClass "-" }} + storageClassName: "" + {{- else }} + storageClassName: {{ $storageClass | quote }} + {{- end }} + {{- else }} + {{- include "weknora.storageClass" . | nindent 2 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/helm/templates/secrets.yaml b/helm/templates/secrets.yaml index e029a10a98..2c259fe9c1 100644 --- a/helm/templates/secrets.yaml +++ b/helm/templates/secrets.yaml @@ -35,10 +35,17 @@ metadata: {{- include "weknora.labels" . | nindent 4 }} type: Opaque stringData: - # Database credentials + {{- if and .Values.database.external (not .Values.database.existingSecret) }} + # External database credentials + DB_USER: {{ required "database.username is required when database.external=true and database.existingSecret is empty" .Values.database.username | quote }} + DB_PASSWORD: {{ required "database.password is required when database.external=true and database.existingSecret is empty" .Values.database.password | quote }} + DB_NAME: {{ required "database.name is required when database.external=true and database.existingSecret is empty" .Values.database.name | quote }} + {{- else if and .Values.postgresql.enabled (not .Values.database.external) (not .Values.mysql.enabled) }} + # PostgreSQL credentials DB_USER: {{ .Values.secrets.dbUser | quote }} DB_PASSWORD: {{ required "secrets.dbPassword is required" .Values.secrets.dbPassword | quote }} DB_NAME: {{ .Values.secrets.dbName | quote }} + {{- end }} # Redis credentials REDIS_USERNAME: {{ .Values.secrets.redisUsername | default "" | quote }} REDIS_PASSWORD: {{ required "secrets.redisPassword is required" .Values.secrets.redisPassword | quote }} diff --git a/helm/values.yaml b/helm/values.yaml index b36ddc6861..da25bb1b5b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -280,6 +280,115 @@ postgresql: # -- Affinity rules affinity: {} +# ----------------------------------------------------------------------------- +# MySQL (Alternative Relational Database) +# ----------------------------------------------------------------------------- +# Mutually exclusive with postgresql.enabled and database.external. +# When enabled, the app's DB_DRIVER is set to "mysql" and DB_HOST to "mysql". +mysql: + # -- Enable MySQL (disables PostgreSQL for the app). Mutually exclusive with + # postgresql.enabled and database.external. + enabled: false + + # -- MySQL image (full reference including repository:tag). + # Defaults match the upstream MySQL 8.0 image. + image: mysql:8.0.37 + + # -- Resource requests and limits + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + + # -- Container security context + securityContext: + allowPrivilegeEscalation: false + + # -- Authentication credentials. When password/rootPassword are empty, random + # values are generated on first install and reused on upgrade (Secret lookup). + # For production, prefer existingSecret so credentials never enter Helm values + # or rendered release manifests. + auth: + # -- Existing Secret with MYSQL_DATABASE, MYSQL_USER, MYSQL_PASSWORD, and + # MYSQL_ROOT_PASSWORD. When set, this chart does not render a MySQL Secret. + existingSecret: "" + # -- Root password (auto-generated if empty) + rootPassword: "" + # -- Application database name + database: WeKnora + # -- Application username + username: weknora + # -- Application password (auto-generated if empty) + password: "" + + # -- Primary instance configuration + primary: + persistence: + # -- Enable persistence + enabled: true + # -- Size of the PVC + size: 20Gi + # -- Storage class name (empty uses cluster default) + storageClass: "" + # -- Use existing PVC (leave empty to create new) + existingClaim: "" + + # -- Node selector + nodeSelector: {} + + # -- Tolerations + tolerations: [] + + # -- Affinity rules + affinity: {} + +# ----------------------------------------------------------------------------- +# External Database +# ----------------------------------------------------------------------------- +# Use an externally managed database by enabling database.external and disabling +# both postgresql.enabled and mysql.enabled. The app pulls DB_DRIVER/HOST/PORT +# from this section. +database: + # -- Use an external database; disable the internal database flags separately + external: false + # -- Database driver: postgres / mysql + driver: postgres + # -- External database host + host: "" + # -- External database port (empty defaults to 5432 for postgres, 3306 for mysql) + port: "" + # -- External database name + name: WeKnora + # -- External database username + username: "" + # -- External database password + password: "" + # -- Name of a pre-existing Secret containing DB_USER, DB_PASSWORD, and + # DB_NAME. When empty, the app uses secrets.existingSecret or the Secret + # rendered by templates/secrets.yaml. + existingSecret: "" + + # -- TLS settings for an external MySQL database. Certificate filenames are + # keys in tls.existingSecret and are mounted read-only under /etc/weknora/mysql-tls. + tls: + # -- Require TLS for the MySQL connection + enabled: false + # -- TLS SNI / certificate verification name + serverName: "" + # -- Skip certificate verification. Development only. + insecureSkipVerify: false + # -- Existing Kubernetes Secret containing certificate files + existingSecret: "" + # -- CA certificate key/filename in existingSecret; empty uses system roots + caFile: "" + # -- Client certificate key/filename in existingSecret + certFile: "" + # -- Client private-key key/filename in existingSecret + keyFile: "" + # ----------------------------------------------------------------------------- # Redis (Stream & Task Queue) # ----------------------------------------------------------------------------- @@ -406,8 +515,12 @@ secrets: # it has been removed from the Secret. systemAesKey: "" - # -- Use existing secret instead of creating one - # The secret must contain keys: DB_USER, DB_PASSWORD, DB_NAME, REDIS_USERNAME, REDIS_PASSWORD, JWT_SECRET, SYSTEM_AES_KEY + # -- Use an existing application Secret instead of rendering one + # Required keys: REDIS_PASSWORD, JWT_SECRET, and SYSTEM_AES_KEY. + # REDIS_USERNAME is optional. DB_USER, DB_PASSWORD, and DB_NAME are required + # for internal PostgreSQL, or for an external database without + # database.existingSecret. NEO4J_USERNAME and NEO4J_PASSWORD are required + # when Neo4j is enabled. existingSecret: "" # ----------------------------------------------------------------------------- @@ -486,7 +599,46 @@ qdrant: enabled: false image: repository: qdrant/qdrant - tag: latest + tag: v1.16.2 + pullPolicy: IfNotPresent + + # -- App connection settings. When enabled=true and host is empty, the app + # connects to the in-chart `qdrant` Service. When enabled=false, host must be + # set if RETRIEVE_DRIVER includes qdrant. + connection: + host: "" + port: 6334 + collection: weknora_embeddings + useTLS: false + + # -- Optional existing Secret containing the Qdrant API key. The same key is + # injected into the in-chart server and the app when enabled. + auth: + existingSecret: "" + apiKeyKey: QDRANT_API_KEY + + service: + type: ClusterIP + httpPort: 6333 + grpcPort: 6334 + + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + + securityContext: + allowPrivilegeEscalation: false + persistence: enabled: true size: 10Gi + storageClass: "" + existingClaim: "" + + nodeSelector: {} + tolerations: [] + affinity: {} diff --git a/internal/agent/tools/database_query.go b/internal/agent/tools/database_query.go index 20bda0b782..5f991f1885 100644 --- a/internal/agent/tools/database_query.go +++ b/internal/agent/tools/database_query.go @@ -115,6 +115,17 @@ func NewDatabaseQueryTool(db *gorm.DB, searchTargets types.SearchTargets) *Datab // Execute executes the database query tool func (t *DatabaseQueryTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) { logger.Infof(ctx, "[Tool][DatabaseQuery] Execute started") + if t.db == nil || t.db.Dialector == nil { + err := fmt.Errorf("database_query is unavailable because the metadata database is not initialized") + return &types.ToolResult{Success: false, Error: err.Error()}, err + } + if dialect := t.db.Dialector.Name(); !DatabaseQuerySupported(dialect) { + err := fmt.Errorf( + "database_query is unavailable for %s metadata databases; use knowledge_search or grep_chunks instead", + dialect, + ) + return &types.ToolResult{Success: false, Error: err.Error()}, err + } tenantID := uint64(0) if tid, ok := ctx.Value(types.TenantIDContextKey).(uint64); ok { @@ -248,6 +259,14 @@ func (t *DatabaseQueryTool) Execute(ctx context.Context, args json.RawMessage) ( }, nil } +// DatabaseQuerySupported reports whether database_query should be exposed for +// the active metadata dialect. SQLite keeps its existing compatibility path; +// MySQL remains disabled because the security pipeline parses and rewrites +// PostgreSQL syntax and has not been validated for MySQL SQL. +func DatabaseQuerySupported(dialect string) bool { + return dialect == "postgres" || dialect == "sqlite" +} + // validateAndSecureSQL validates the SQL query and injects tenant_id conditions func (t *DatabaseQueryTool) validateAndSecureSQL(sqlQuery string, tenantID uint64) (string, error) { searchScopes := searchScopesFromTargets(t.searchTargets) diff --git a/internal/agent/tools/database_query_dialect_test.go b/internal/agent/tools/database_query_dialect_test.go new file mode 100644 index 0000000000..5af25258e2 --- /dev/null +++ b/internal/agent/tools/database_query_dialect_test.go @@ -0,0 +1,52 @@ +package tools + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/require" + "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +func TestDatabaseQueryToolRejectsMySQLBeforeParsingOrExecutingSQL(t *testing.T) { + sqlDB, mock, err := sqlmock.New() + require.NoError(t, err) + t.Cleanup(func() { _ = sqlDB.Close() }) + + db, err := gorm.Open(mysql.New(mysql.Config{ + Conn: sqlDB, + SkipInitializeWithVersion: true, + }), &gorm.Config{}) + require.NoError(t, err) + + tool := NewDatabaseQueryTool(db, nil) + result, err := tool.Execute(context.Background(), json.RawMessage(`{"sql":"SELECT id FROM knowledge_bases"}`)) + require.Error(t, err) + require.NotNil(t, result) + require.False(t, result.Success) + require.True(t, strings.Contains(strings.ToLower(result.Error), "mysql")) + require.True(t, strings.Contains(strings.ToLower(result.Error), "unavailable")) + require.NoError(t, mock.ExpectationsWereMet(), "unsupported dialect must not reach the database") +} + +func TestDatabaseQuerySupportedDialects(t *testing.T) { + tests := []struct { + dialect string + want bool + }{ + {dialect: "postgres", want: true}, + {dialect: "sqlite", want: true}, + {dialect: "mysql", want: false}, + {dialect: "", want: false}, + } + + for _, testCase := range tests { + t.Run(testCase.dialect, func(t *testing.T) { + require.Equal(t, testCase.want, DatabaseQuerySupported(testCase.dialect)) + }) + } +} diff --git a/internal/agent/tools/grep_chunks.go b/internal/agent/tools/grep_chunks.go index 9f5dc05e42..7ef050d482 100644 --- a/internal/agent/tools/grep_chunks.go +++ b/internal/agent/tools/grep_chunks.go @@ -18,15 +18,21 @@ import ( var grepChunksTool = BaseTool{ name: ToolGrepChunks, - description: `Search knowledge base chunk content with a single POSIX regular expression, applied directly in the database (PostgreSQL ~* / MySQL/SQLite REGEXP, case-insensitive). Behaves like ` + "`grep -E -i`" + `. + description: `Search knowledge base chunk content with a single case-insensitive regular expression. +Behaves like ` + "`grep -E -i`" + `. Pack multiple concepts into ONE regex using ` + "`|`" + ` alternation — do not call this tool repeatedly for synonyms. Returns matching chunks with a short cN chunk source ID, a parent dN document ID, and a snippet around the first match. Examples: - Alternation (RECOMMENDED): "stardust|skyvault|psionic" (matches any of the words) - Multiple terms in order: "psionic.*engine" (matches both words in order) -- Word boundary / anchor: "\\brag\\b" or "^chapter\\s+\\d+" +- Anchors and character classes: "^chapter\\s+\\d+" - Plain text: "engine" (matches literal substring anywhere in chunk content) -IMPORTANT — JSON escaping: every backslash in a regex MUST be written as \\ inside the JSON tool arguments (e.g. to search for literal "C++" write "C\\+\\+", NOT "C\+\+"; for "\d+" write "\\d+"). Plain "\+" / "\d" etc. are invalid JSON escapes and will fail to parse. +Use the portable syntax shared by supported databases: literals, character ranges and classes +(\d \D \s \S \w \W), grouping, alternation, anchors, and quantifiers. Do not use \b (it means a word +boundary on some databases and a literal backspace on others), other letter escapes, or constructs beginning with "(?". +IMPORTANT — JSON escaping: every backslash in a regex MUST be written as \\ inside the JSON tool arguments. +For example, to search for literal "C++" write "C\\+\\+", NOT "C\+\+". +Plain "\+" is an invalid JSON escape and will fail to parse. Use this to locate candidate chunks by exact identifiers, error codes, product names, or recurring terms. ## Deep read after grep: @@ -37,7 +43,9 @@ Use this to locate candidate chunks by exact identifiers, error codes, product n "properties": { "query": { "type": "string", - "description": "A single POSIX regex applied directly to chunk content (case-insensitive). Combine multiple concepts with \"|\" alternation in ONE regex (e.g. \"stardust|skyvault|psionic\") — do not split into multiple calls.", + "description": "A single case-insensitive regex applied directly to chunk content. ` + + `Combine multiple concepts with \"|\" alternation in ONE regex ` + + `(e.g. \"stardust|skyvault|psionic\"); do not split into multiple calls.", "minLength": 1 } }, @@ -55,9 +63,8 @@ type GrepChunksInput struct { Query string `json:"query,omitempty"` } -// GrepChunksTool performs regex pattern matching across knowledge base chunks. -// PostgreSQL: uses the case-insensitive POSIX operator ~*. -// MySQL/SQLite: falls back to REGEXP. +// GrepChunksTool performs case-insensitive regex pattern matching across +// knowledge base chunks. // // The tool tracks previously-returned chunk IDs per-instance (one instance per // agent session) so that a subsequent search hitting the same chunk can be @@ -109,9 +116,9 @@ func (t *GrepChunksTool) Execute(ctx context.Context, args json.RawMessage) (*ty }, fmt.Errorf("missing query parameter") } - // Compile with (?i) prefix for case-insensitive Go-side matching. - // Compilation also validates the regex syntax before we send it to the DB. - re, err := regexp.Compile("(?i)" + query) + // Compile once for Go-side scoring and validate the cross-database subset + // before sending the same pattern to the active SQL regex engine. + re, err := compilePortableCaseInsensitiveRegex(query) if err != nil { logger.Errorf(ctx, "[Tool][GrepChunks] Invalid regex %q: %v", query, err) return &types.ToolResult{ @@ -236,18 +243,20 @@ type chunkWithTitle struct { TotalChunkCount int `json:"total_chunk_count" gorm:"column:total_chunk_count"` } -// regexOperatorForDialect returns the SQL operator used to apply a POSIX -// regular expression to a text column for the current dialect. -// PostgreSQL ~* is case-insensitive by default; MySQL/SQLite REGEXP relies on -// collation / driver extensions. -func (t *GrepChunksTool) regexOperatorForDialect() string { +// caseInsensitiveRegexForDialect returns the SQL predicate used to apply a +// case-insensitive regular expression to one text column. +func (t *GrepChunksTool) caseInsensitiveRegexForDialect(column string) (string, error) { switch t.db.Dialector.Name() { case "postgres": - return "~*" + return column + " ~* ?", nil + case "mysql": + // REGEXP_LIKE's explicit match flag keeps behavior independent of the + // column or connection collation. + return "REGEXP_LIKE(" + column + ", ?, 'i')", nil + case "sqlite": + return column + " REGEXP ?", nil default: - // MySQL, SQLite (with the go-sqlite3 REGEXP extension), or anything else - // that understands the REGEXP keyword. - return "REGEXP" + return "", fmt.Errorf("unsupported database dialect %q for regex search", t.db.Dialector.Name()) } } @@ -380,8 +389,6 @@ func (t *GrepChunksTool) searchChunks( return nil, nil } - regexOp := t.regexOperatorForDialect() - query := t.db.WithContext(ctx).Table("chunks"). Select("chunks.id, chunks.content, chunks.chunk_index, chunks.knowledge_id, "+ "chunks.knowledge_base_id, chunks.chunk_type, chunks.metadata, chunks.created_at, "+ @@ -403,9 +410,14 @@ func (t *GrepChunksTool) searchChunks( len(knowledgeIDs), len(tagTargets), len(kbIDs)) query = query.Where(scopeSQL, scopeArgs...) - // For MySQL/SQLite REGEXP case-insensitivity we rely on the column's default - // collation (utf8mb4_general_ci etc.) OR the driver's REGEXP implementation, - // which mirrors what wiki_search already ships in this codebase. + contentRegex, err := t.caseInsensitiveRegexForDialect("chunks.content") + if err != nil { + return nil, err + } + titleRegex, err := t.caseInsensitiveRegexForDialect("knowledges.title") + if err != nil { + return nil, err + } var regexConditions []string var regexArgs []interface{} for _, q := range queries { @@ -413,7 +425,7 @@ func (t *GrepChunksTool) searchChunks( // knowledge's title, so a doc whose title matches (e.g. titled // "图片素材") surfaces even when its body rarely repeats the term. regexConditions = append(regexConditions, - fmt.Sprintf("(chunks.content %s ? OR knowledges.title %s ?)", regexOp, regexOp)) + "("+contentRegex+" OR "+titleRegex+")") regexArgs = append(regexArgs, q, q) } query = query.Where("("+strings.Join(regexConditions, " OR ")+")", regexArgs...) diff --git a/internal/agent/tools/portable_regex.go b/internal/agent/tools/portable_regex.go new file mode 100644 index 0000000000..e9e753de11 --- /dev/null +++ b/internal/agent/tools/portable_regex.go @@ -0,0 +1,82 @@ +package tools + +import ( + "fmt" + "regexp" + "unicode" +) + +// compilePortableCaseInsensitiveRegex limits database-backed regex tools to +// syntax shared by PostgreSQL ARE, MySQL ICU, and Go RE2. Engine-specific +// shorthand and inline constructs otherwise make validation and DB execution +// disagree across supported deployments. +func compilePortableCaseInsensitiveRegex(pattern string) (*regexp.Regexp, error) { + if err := validatePortableRegex(pattern); err != nil { + return nil, err + } + + compiled, err := regexp.Compile("(?i:" + pattern + ")") + if err != nil { + return nil, fmt.Errorf("invalid portable regular expression %q: %w", pattern, err) + } + return compiled, nil +} + +// portableEscapes are the backslash escapes that PostgreSQL ARE, MySQL ICU, +// and Go RE2 all agree on, so a pattern using them matches the same text +// whichever backend runs it. +// +// Notably absent is \b: MySQL and RE2 read it as a word boundary, while +// PostgreSQL ARE reads it as a literal backspace, so the same pattern silently +// matches different text per deployment. \A, \Z, \z, \m, \M, \y and numeric +// backreferences diverge the same way and stay rejected. +var portableEscapes = map[rune]struct{}{ + 'd': {}, 'D': {}, + 's': {}, 'S': {}, + 'w': {}, 'W': {}, + 'n': {}, 'r': {}, 't': {}, 'f': {}, +} + +func validatePortableRegex(pattern string) error { + runes := []rune(pattern) + inCharacterClass := false + + for index := 0; index < len(runes); index++ { + current := runes[index] + if current == '\\' { + if index+1 >= len(runes) { + return fmt.Errorf("invalid portable regular expression %q: trailing backslash", pattern) + } + escaped := runes[index+1] + _, portable := portableEscapes[escaped] + if !portable && (unicode.IsLetter(escaped) || unicode.IsDigit(escaped)) { + return fmt.Errorf( + "non-portable regex escape \\%c in %q; portable escapes are "+ + "\\d \\D \\s \\S \\w \\W \\n \\r \\t \\f, plus literals, character "+ + "ranges, grouping, alternation, anchors, and quantifiers", + escaped, + pattern, + ) + } + index++ + continue + } + + switch current { + case '[': + inCharacterClass = true + case ']': + inCharacterClass = false + case '(': + if !inCharacterClass && index+1 < len(runes) && runes[index+1] == '?' { + return fmt.Errorf( + "non-portable regex construct \"(?\" in %q; inline flags, lookarounds, "+ + "and named groups are not supported", + pattern, + ) + } + } + } + + return nil +} diff --git a/internal/agent/tools/portable_regex_test.go b/internal/agent/tools/portable_regex_test.go new file mode 100644 index 0000000000..bb401d9ef6 --- /dev/null +++ b/internal/agent/tools/portable_regex_test.go @@ -0,0 +1,92 @@ +package tools + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestDatabaseRegexToolsRejectNonPortableSyntax(t *testing.T) { + tests := []struct { + name string + execute func() (*types.ToolResult, error) + }{ + { + name: "grep_chunks", + execute: func() (*types.ToolResult, error) { + return NewGrepChunksTool(nil, nil).Execute( + context.Background(), + json.RawMessage(`{"query":"\\brag\\b"}`), + ) + }, + }, + { + name: "wiki_search", + execute: func() (*types.ToolResult, error) { + return NewWikiSearchTool(nil, nil, nil, nil).Execute( + context.Background(), + json.RawMessage(`{"queries":["\\brag\\b"]}`), + ) + }, + }, + } + + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + result, err := testCase.execute() + if err == nil { + t.Fatal("expected non-portable regex to return an error") + } + if result == nil || result.Success { + t.Fatalf("expected failed tool result, got %#v", result) + } + if !strings.Contains(result.Error, "portable") { + t.Fatalf("expected actionable portable-regex error, got %q", result.Error) + } + }) + } +} + +// The character-class shorthands mean the same thing to PostgreSQL ARE, MySQL +// ICU and Go RE2, so rejecting them only cost the agent expressiveness without +// buying any cross-database consistency. +func TestPortableRegexAcceptsAgreedCharacterClasses(t *testing.T) { + for _, pattern := range []string{ + `\d+`, + `^chapter\s+\d+`, + `\w+`, + `[A-Z]\S*`, + `error\D`, + `a\tb`, + `C\+\+`, + } { + if _, err := compilePortableCaseInsensitiveRegex(pattern); err != nil { + t.Errorf("pattern %q should be portable, got error: %v", pattern, err) + } + } +} + +// \b is the motivating case for keeping a portable subset at all: MySQL and +// RE2 read it as a word boundary while PostgreSQL ARE reads it as a literal +// backspace, so the same pattern matches different text per deployment. +func TestPortableRegexRejectsEscapesThatDivergeAcrossEngines(t *testing.T) { + for _, pattern := range []string{ + `\brag\b`, + `\Bfoo`, + `\mword`, + `\yword`, + `\Astart`, + `\Zend`, + `(foo)\1`, + `(?i)foo`, + `(?=foo)`, + `trailing\`, + } { + if _, err := compilePortableCaseInsensitiveRegex(pattern); err == nil { + t.Errorf("pattern %q should be rejected as non-portable", pattern) + } + } +} diff --git a/internal/agent/tools/wiki_tools.go b/internal/agent/tools/wiki_tools.go index 877f020f03..45b80eb7e0 100644 --- a/internal/agent/tools/wiki_tools.go +++ b/internal/agent/tools/wiki_tools.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "regexp" "strings" "sync" "unicode/utf8" @@ -771,7 +770,7 @@ func NewWikiSearchTool( return &wikiSearchTool{ BaseTool: NewBaseTool( ToolWikiSearch, - `Search wiki pages using PostgreSQL POSIX regular expressions (~* operator, case-insensitive). + `Search wiki pages using case-insensitive regular expressions. STRONGLY PREFER using regex to search for multiple concepts at once rather than simple plain text queries. Returns matching pages with titles, slugs, and summaries (each tagged with its short bN knowledge_base_id). Examples: @@ -779,7 +778,12 @@ Examples: - Multiple terms (RECOMMENDED): "psionic.*engine" (matches both words in order) - Prefix matching: "^entity/.*" (finds all entities) - Plain text: "engine" (matches anywhere in title/content/slug/summary) -IMPORTANT — JSON escaping: every backslash in a regex MUST be written as \\ inside the JSON tool arguments (e.g. to search for literal "C++" write "C\\+\\+", NOT "C\+\+"; for "\d+" write "\\d+"). Plain "\+" / "\d" etc. are invalid JSON escapes and will fail to parse. +Use the portable syntax shared by supported databases: literals, character ranges and classes +(\d \D \s \S \w \W), grouping, alternation, anchors, and quantifiers. Do not use \b (it means a word +boundary on some databases and a literal backspace on others), other letter escapes, or constructs beginning with "(?". +IMPORTANT — JSON escaping: every backslash in a regex MUST be written as \\ inside the JSON tool arguments. +For example, to search for literal "C++" write "C\\+\\+", NOT "C\+\+". +Plain "\+" is an invalid JSON escape and will fail to parse. Use this to find relevant wiki pages when you don't know the exact slug.`, json.RawMessage(`{ "type": "object", @@ -827,6 +831,11 @@ func (t *wikiSearchTool) Execute(ctx context.Context, args json.RawMessage) (*ty if len(queriesToRun) == 0 { return &types.ToolResult{Success: false, Error: "Missing 'queries' parameter"}, nil } + for _, query := range queriesToRun { + if _, err := compilePortableCaseInsensitiveRegex(query); err != nil { + return &types.ToolResult{Success: false, Error: err.Error()}, err + } + } if params.Limit <= 0 { params.Limit = 10 @@ -1039,7 +1048,7 @@ func extractSnippet(content string, query string) string { if content == "" || query == "" { return "" } - re, err := regexp.Compile("(?i)" + query) + re, err := compilePortableCaseInsensitiveRegex(query) if err != nil { return "" } diff --git a/internal/agent/tools/wiki_tools_test.go b/internal/agent/tools/wiki_tools_test.go index 998a3817a5..1c54965589 100644 --- a/internal/agent/tools/wiki_tools_test.go +++ b/internal/agent/tools/wiki_tools_test.go @@ -159,6 +159,24 @@ func TestWikiToolsInAvailableDefinitions(t *testing.T) { } } +func TestWikiSearchDescriptionIsDatabaseAgnostic(t *testing.T) { + tool := NewWikiSearchTool(nil, nil, nil, nil) + description := tool.Description() + + for _, databaseSpecificTerm := range []string{"PostgreSQL", "~* operator"} { + if strings.Contains(description, databaseSpecificTerm) { + t.Fatalf( + "wiki_search description must not expose database-specific syntax %q: %s", + databaseSpecificTerm, + description, + ) + } + } + if !strings.Contains(description, "case-insensitive regular expressions") { + t.Fatalf("wiki_search description must document its portable regex behavior: %s", description) + } +} + func TestWikiReadPageSchemaUsesAutomaticKBRouting(t *testing.T) { tool := NewWikiReadPageTool( &fakeWikiPageService{}, nil, diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go index 08723f3b64..0cd5e2cb12 100644 --- a/internal/application/repository/chunk.go +++ b/internal/application/repository/chunk.go @@ -136,10 +136,44 @@ func (r *chunkRepository) ListChunksBySeqID( if len(seqIDs) == 0 { return []*types.Chunk{}, nil } + uniqueSeqIDs := make([]int64, 0, len(seqIDs)) + seen := make(map[int64]struct{}, len(seqIDs)) + for _, seqID := range seqIDs { + if _, exists := seen[seqID]; exists { + continue + } + seen[seqID] = struct{}{} + uniqueSeqIDs = append(uniqueSeqIDs, seqID) + } + + const batchSize = 5000 var chunks []*types.Chunk - if err := r.db.WithContext(ctx). - Where("tenant_id = ? AND seq_id IN ?", tenantID, seqIDs). - Find(&chunks).Error; err != nil { + if len(uniqueSeqIDs) <= batchSize { + if err := r.db.WithContext(ctx). + Where("tenant_id = ? AND seq_id IN ?", tenantID, uniqueSeqIDs). + Find(&chunks).Error; err != nil { + return nil, err + } + return chunks, nil + } + + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + for start := 0; start < len(uniqueSeqIDs); start += batchSize { + end := start + batchSize + if end > len(uniqueSeqIDs) { + end = len(uniqueSeqIDs) + } + var batch []*types.Chunk + if err := tx. + Where("tenant_id = ? AND seq_id IN ?", tenantID, uniqueSeqIDs[start:end]). + Find(&batch).Error; err != nil { + return err + } + chunks = append(chunks, batch...) + } + return nil + }) + if err != nil { return nil, err } return chunks, nil @@ -397,7 +431,29 @@ func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chun if len(chunks) == 0 { return nil } + const batchSize = 5000 + if len(chunks) <= batchSize { + return r.updateChunksBatch(ctx, chunks) + } + + // Keep the caller-visible operation atomic even though MySQL's prepared + // statement protocol limits one statement to 65,535 placeholders. + return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + batchRepo := &chunkRepository{db: tx} + for start := 0; start < len(chunks); start += batchSize { + end := start + batchSize + if end > len(chunks) { + end = len(chunks) + } + if err := batchRepo.updateChunksBatch(ctx, chunks[start:end]); err != nil { + return err + } + } + return nil + }) +} +func (r *chunkRepository) updateChunksBatch(ctx context.Context, chunks []*types.Chunk) error { // Build batch update SQL with CASE expressions var ids []string contentCases := make([]string, 0, len(chunks)) @@ -412,6 +468,9 @@ func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chun var flagsArgs []interface{} var statusArgs []interface{} + dialectName := r.db.Dialector.Name() + isPostgres := dialectName == "postgres" + for _, chunk := range chunks { ids = append(ids, chunk.ID) content := common.CleanInvalidUTF8(chunk.Content) @@ -419,22 +478,47 @@ func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chun contentCases = append(contentCases, "WHEN id = ? THEN ?") contentArgs = append(contentArgs, chunk.ID, content) - // Convert bool to string for PostgreSQL compatibility - isEnabledStr := "false" - if chunk.IsEnabled { - isEnabledStr = "true" + // PostgreSQL accepts 'true'/'false' string literals in a + // boolean context (and the SQL below casts with ::boolean). MySQL + // under STRICT_TRANS_TABLES rejects string->BOOLEAN coercion with + // Error 1292, so pass an integer 0/1 there. SQLite accepts both. + var isEnabledVal interface{} + if isPostgres { + if chunk.IsEnabled { + isEnabledVal = "true" + } else { + isEnabledVal = "false" + } + } else { + if chunk.IsEnabled { + isEnabledVal = 1 + } else { + isEnabledVal = 0 + } } isEnabledCases = append(isEnabledCases, "WHEN id = ? THEN ?") - isEnabledArgs = append(isEnabledArgs, chunk.ID, isEnabledStr) + isEnabledArgs = append(isEnabledArgs, chunk.ID, isEnabledVal) tagIDCases = append(tagIDCases, "WHEN id = ? THEN ?") tagIDArgs = append(tagIDArgs, chunk.ID, chunk.TagID) + // flags / status are INTEGER columns. PostgreSQL tolerates a + // text literal because the SQL casts with ::integer, but MySQL + // strict mode rejects it (Error 1292). Pass native int32 for + // non-PostgreSQL dialects. + var flagsVal interface{} + var statusVal interface{} + if isPostgres { + flagsVal = fmt.Sprintf("%d", chunk.Flags) + statusVal = fmt.Sprintf("%d", chunk.Status) + } else { + flagsVal = chunk.Flags + statusVal = chunk.Status + } flagsCases = append(flagsCases, "WHEN id = ? THEN ?") - flagsArgs = append(flagsArgs, chunk.ID, fmt.Sprintf("%d", chunk.Flags)) - + flagsArgs = append(flagsArgs, chunk.ID, flagsVal) statusCases = append(statusCases, "WHEN id = ? THEN ?") - statusArgs = append(statusArgs, chunk.ID, fmt.Sprintf("%d", chunk.Status)) + statusArgs = append(statusArgs, chunk.ID, statusVal) } // Build IN clause placeholders @@ -454,46 +538,43 @@ func (r *chunkRepository) UpdateChunks(ctx context.Context, chunks []*types.Chun args = append(args, id) } - isPostgres := r.db.Dialector.Name() == "postgres" - - var sql string + // One SQL template, dialect-aware bits hoisted into small variables: + // - Postgres needs ::boolean / ::integer casts on CASE results + // (the bind values are string literals). + // - Postgres and MySQL both use NOW() for the timestamp; SQLite has + // neither NOW() nor casts, so it uses datetime('now'). + // - The bind values are already shaped per-dialect above (string + // for Postgres, native int for MySQL/SQLite), so the only thing + // that differs in the SQL is the cast suffix and timestamp expr. + boolCast, intCast := "", "" if isPostgres { - sql = fmt.Sprintf(` - UPDATE chunks SET - content = CASE %s END, - is_enabled = (CASE %s END)::boolean, - tag_id = CASE %s END, - flags = (CASE %s END)::integer, - status = (CASE %s END)::integer, - updated_at = NOW() - WHERE id IN (%s) - `, - strings.Join(contentCases, " "), - strings.Join(isEnabledCases, " "), - strings.Join(tagIDCases, " "), - strings.Join(flagsCases, " "), - strings.Join(statusCases, " "), - strings.Join(inPlaceholders, ","), - ) - } else { - sql = fmt.Sprintf(` - UPDATE chunks SET - content = CASE %s END, - is_enabled = CASE %s END, - tag_id = CASE %s END, - flags = CASE %s END, - status = CASE %s END, - updated_at = datetime('now') - WHERE id IN (%s) - `, - strings.Join(contentCases, " "), - strings.Join(isEnabledCases, " "), - strings.Join(tagIDCases, " "), - strings.Join(flagsCases, " "), - strings.Join(statusCases, " "), - strings.Join(inPlaceholders, ","), - ) + boolCast, intCast = "::boolean", "::integer" + } + tsExpr := "NOW()" + if dialectName == "mysql" { + tsExpr = "NOW(6)" // microsecond precision to match DATETIME(6) columns + } + if dialectName == "sqlite" { + tsExpr = "datetime('now')" } + sql := fmt.Sprintf(` + UPDATE chunks SET + content = CASE %s END, + is_enabled = (CASE %s END)%s, + tag_id = CASE %s END, + flags = (CASE %s END)%s, + status = (CASE %s END)%s, + updated_at = %s + WHERE id IN (%s) + `, + strings.Join(contentCases, " "), + strings.Join(isEnabledCases, " "), boolCast, + strings.Join(tagIDCases, " "), + strings.Join(flagsCases, " "), intCast, + strings.Join(statusCases, " "), intCast, + tsExpr, + strings.Join(inPlaceholders, ","), + ) return r.db.WithContext(ctx).Exec(sql, args...).Error } @@ -852,7 +933,46 @@ func (r *chunkRepository) UpdateChunkFlagsBatch( if len(allIDs) == 0 { return nil } + const batchSize = 5000 + if len(allIDs) > batchSize { + return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + batchRepo := &chunkRepository{db: tx} + for start := 0; start < len(allIDs); start += batchSize { + end := start + batchSize + if end > len(allIDs) { + end = len(allIDs) + } + batchSet := make(map[string]types.ChunkFlags, end-start) + batchClear := make(map[string]types.ChunkFlags, end-start) + for _, id := range allIDs[start:end] { + if flag, ok := setFlags[id]; ok { + batchSet[id] = flag + } + if flag, ok := clearFlags[id]; ok { + batchClear[id] = flag + } + } + if err := batchRepo.updateChunkFlagsBatch( + ctx, tenantID, kbID, batchSet, batchClear, allIDs[start:end], + ); err != nil { + return err + } + } + return nil + }) + } + return r.updateChunkFlagsBatch(ctx, tenantID, kbID, setFlags, clearFlags, allIDs) +} + +func (r *chunkRepository) updateChunkFlagsBatch( + ctx context.Context, + tenantID uint64, + kbID string, + setFlags map[string]types.ChunkFlags, + clearFlags map[string]types.ChunkFlags, + allIDs []string, +) error { // Build CASE expression for flags update // flags = (flags | setFlag) & ~clearFlag var setCases, clearCases []string @@ -888,6 +1008,9 @@ func (r *chunkRepository) UpdateChunkFlagsBatch( } nowFunc := "NOW()" + if r.db.Dialector.Name() == "mysql" { + nowFunc = "NOW(6)" + } if r.db.Dialector.Name() == "sqlite" { nowFunc = "datetime('now')" } @@ -922,12 +1045,19 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID( newTagID *string, excludeIDs []string, ) ([]string, error) { - // First, get the IDs of chunks that will be affected (for is_enabled sync) + const maxInlineExclusions = 5000 + if len(excludeIDs) > maxInlineExclusions { + return r.updateChunkFieldsByTagIDWithLargeExclusionSet( + ctx, tenantID, kbID, tagID, isEnabled, setFlags, clearFlags, newTagID, excludeIDs, + ) + } + + // First, get the IDs whose retriever-visible fields will change. var affectedIDs []string - if isEnabled != nil { + if isEnabled != nil || newTagID != nil { var chunks []*types.Chunk query := r.db.WithContext(ctx). - Select("id"). + Select("id, is_enabled, tag_id"). Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", tenantID, kbID, types.ChunkTypeFAQ) if tagID != "" { @@ -938,8 +1068,17 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID( query = query.Where("id NOT IN ?", excludeIDs) } - // Only get chunks that need to change - query = query.Where("is_enabled != ?", *isEnabled) + var changeClauses []string + var changeArgs []interface{} + if isEnabled != nil { + changeClauses = append(changeClauses, "is_enabled != ?") + changeArgs = append(changeArgs, *isEnabled) + } + if newTagID != nil { + changeClauses = append(changeClauses, "tag_id != ?") + changeArgs = append(changeArgs, *newTagID) + } + query = query.Where("("+strings.Join(changeClauses, " OR ")+")", changeArgs...) if err := query.Find(&chunks).Error; err != nil { return nil, err } @@ -983,7 +1122,7 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID( if clearFlags != 0 { flagsExpr = fmt.Sprintf("(%s & ~%d)", flagsExpr, int(clearFlags)) } - updates["flags"] = r.db.Raw(flagsExpr) + updates["flags"] = gorm.Expr(flagsExpr) } if err := query.Updates(updates).Error; err != nil { @@ -1088,6 +1227,89 @@ func diffFAQChunkIDsByContentHash(src, dst []chunkIDHash) ( return chunksToAdd, chunksToDelete, matched } +func (r *chunkRepository) updateChunkFieldsByTagIDWithLargeExclusionSet( + ctx context.Context, + tenantID uint64, + kbID string, + tagID string, + isEnabled *bool, + setFlags types.ChunkFlags, + clearFlags types.ChunkFlags, + newTagID *string, + excludeIDs []string, +) ([]string, error) { + excluded := make(map[string]struct{}, len(excludeIDs)) + for _, id := range excludeIDs { + excluded[id] = struct{}{} + } + + var affectedIDs []string + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var candidates []*types.Chunk + query := tx.Select("id, is_enabled, tag_id"). + Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", + tenantID, kbID, types.ChunkTypeFAQ) + if tagID != "" { + query = query.Where("tag_id = ?", tagID) + } + if err := query.Find(&candidates).Error; err != nil { + return err + } + + includedIDs := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + if _, skip := excluded[candidate.ID]; skip { + continue + } + includedIDs = append(includedIDs, candidate.ID) + if (isEnabled != nil && candidate.IsEnabled != *isEnabled) || + (newTagID != nil && candidate.TagID != *newTagID) { + affectedIDs = append(affectedIDs, candidate.ID) + } + } + if len(includedIDs) == 0 { + return nil + } + + updates := map[string]interface{}{"updated_at": time.Now().UTC()} + if isEnabled != nil { + updates["is_enabled"] = *isEnabled + } + if newTagID != nil { + updates["tag_id"] = *newTagID + } + if setFlags != 0 || clearFlags != 0 { + flagsExpr := "flags" + if setFlags != 0 { + flagsExpr = fmt.Sprintf("(%s | %d)", flagsExpr, int(setFlags)) + } + if clearFlags != 0 { + flagsExpr = fmt.Sprintf("(%s & ~%d)", flagsExpr, int(clearFlags)) + } + updates["flags"] = gorm.Expr(flagsExpr) + } + + const batchSize = 5000 + for start := 0; start < len(includedIDs); start += batchSize { + end := start + batchSize + if end > len(includedIDs) { + end = len(includedIDs) + } + if err := tx.Model(&types.Chunk{}). + Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ? AND id IN ?", + tenantID, kbID, types.ChunkTypeFAQ, includedIDs[start:end]). + Updates(updates).Error; err != nil { + return err + } + } + return nil + }) + if err != nil { + return nil, err + } + return affectedIDs, nil +} + // FAQChunkDiff compares FAQ chunks between two knowledge bases and returns the differences. // Returns: chunksToAdd (IDs of chunks in src whose content_hash is not in dst), // diff --git a/internal/application/repository/chunk_sqlite_test.go b/internal/application/repository/chunk_sqlite_test.go index 34163c8194..21db0eaa26 100644 --- a/internal/application/repository/chunk_sqlite_test.go +++ b/internal/application/repository/chunk_sqlite_test.go @@ -3,6 +3,7 @@ package repository import ( "context" "testing" + "time" "github.com/Tencent/WeKnora/internal/types" "github.com/google/uuid" @@ -331,3 +332,46 @@ func TestListRecentDocumentChunksWithQuestions_UnionsExplicitKBAndKnowledge(t *t require.Len(t, got, 2) assert.ElementsMatch(t, []string{fromExplicitKB.ID, fromExplicitDocument.ID}, []string{got[0].ID, got[1].ID}) } + +// TestUpdateChunks_SQLite_RewritesFieldsAndTimestamp exercises the +// SQLite branch of UpdateChunks (datetime('now')) so the three-dialect +// switch has a red-capable unit test that does not need a live DB. +func TestUpdateChunks_SQLite_RewritesFieldsAndTimestamp(t *testing.T) { + db := setupChunkTestDB(t) + repo := NewChunkRepository(db) + ctx := context.Background() + + kbID := uuid.New().String() + knowledgeID := uuid.New().String() + c1 := makeChunk(kbID, knowledgeID, "text") + c2 := makeChunk(kbID, knowledgeID, "text") + require.NoError(t, repo.CreateChunks(ctx, []*types.Chunk{c1, c2})) + oldTime := time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC) + result := db.Model(&types.Chunk{}). + Where("id IN ?", []string{c1.ID, c2.ID}). + UpdateColumn("updated_at", oldTime) + require.NoError(t, result.Error) + require.Equal(t, int64(2), result.RowsAffected) + + // Mutate exactly the fields UpdateChunks rewrites. + c1.Content = "updated content 1" + c1.Status = 2 + c1.IsEnabled = false + c2.Content = "updated content 2" + c2.Status = 3 + + require.NoError(t, repo.UpdateChunks(ctx, []*types.Chunk{c1, c2})) + + var got1, got2 types.Chunk + require.NoError(t, db.First(&got1, "id = ?", c1.ID).Error) + require.NoError(t, db.First(&got2, "id = ?", c2.ID).Error) + + assert.Equal(t, "updated content 1", got1.Content) + assert.Equal(t, 2, int(got1.Status)) + assert.False(t, got1.IsEnabled) + assert.Equal(t, "updated content 2", got2.Content) + assert.Equal(t, 3, int(got2.Status)) + + assert.True(t, got1.UpdatedAt.After(oldTime), "first chunk updated_at must advance") + assert.True(t, got2.UpdatedAt.After(oldTime), "second chunk updated_at must advance") +} diff --git a/internal/application/repository/datasource_repo.go b/internal/application/repository/datasource_repo.go index 4771aa1530..fe40a82473 100644 --- a/internal/application/repository/datasource_repo.go +++ b/internal/application/repository/datasource_repo.go @@ -298,9 +298,9 @@ func (r *SyncLogRepository) CleanupOldLogs(ctx context.Context, retentionDays in if retentionDays <= 0 { retentionDays = 30 } - // Delete logs older than the retention period + cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays) if err := r.db.WithContext(ctx). - Where("started_at < NOW() - INTERVAL ? DAY", retentionDays). + Where("started_at < ?", cutoff). Delete(&types.SyncLog{}).Error; err != nil { return err } diff --git a/internal/application/repository/datasource_repo_test.go b/internal/application/repository/datasource_repo_test.go index e3ab5c2607..3b7c34a24c 100644 --- a/internal/application/repository/datasource_repo_test.go +++ b/internal/application/repository/datasource_repo_test.go @@ -122,3 +122,34 @@ func TestSyncLogRepositoryUpdateResultClearsErrorMessage(t *testing.T) { assert.Equal(t, result.ToString(), stored.Result.ToString()) require.NotNil(t, stored.FinishedAt) } + +func TestSyncLogRepositoryCleanupOldLogsUsesPortableCutoff(t *testing.T) { + db := setupDataSourceRepoTestDB(t) + repo := NewSyncLogRepository(db) + now := time.Now().UTC() + + for _, log := range []*types.SyncLog{ + { + ID: "old-log", + DataSourceID: "ds-1", + TenantID: 1, + Status: types.SyncLogStatusSuccess, + StartedAt: now.Add(-48 * time.Hour), + }, + { + ID: "recent-log", + DataSourceID: "ds-1", + TenantID: 1, + Status: types.SyncLogStatusSuccess, + StartedAt: now.Add(-time.Hour), + }, + } { + require.NoError(t, repo.Create(context.Background(), log)) + } + + require.NoError(t, repo.CleanupOldLogs(context.Background(), 1)) + + var ids []string + require.NoError(t, db.Model(&types.SyncLog{}).Order("id").Pluck("id", &ids).Error) + assert.Equal(t, []string{"recent-log"}, ids) +} diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go index 9c17687b79..552cc12725 100644 --- a/internal/application/repository/knowledge.go +++ b/internal/application/repository/knowledge.go @@ -3,9 +3,11 @@ package repository import ( "context" "errors" + "fmt" "strings" "time" + "github.com/Tencent/WeKnora/internal/database" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "gorm.io/gorm" @@ -575,12 +577,18 @@ func (r *knowledgeRepository) CountKnowledgeByStatus( return count, nil } -// SearchKnowledge searches knowledge items by keyword across the tenant -// If keyword is empty, returns recent files -// Only returns documents from document-type knowledge bases (excludes FAQ) -// Returns (results, hasMore, error) -// FindByMetadataKey finds a knowledge item by a key-value pair in the metadata JSON column. -// Uses Postgres jsonb operator: metadata->>'key' = 'value'. +// FindByMetadataKey finds a knowledge item by a key-value pair in the +// metadata JSON column. The JSON extraction syntax is dialect-aware via +// database.JSONPathExpr: +// +// - postgres: metadata ->> 'key' +// - mysql: metadata ->> '$.key' (bare-key form errors 1064) +// - sqlite: json_extract(metadata, '$.key') +// +// Used by the datasource sync loop to deduplicate by external_id; a +// query error here MUST NOT be swallowed by the caller - the +// datasource service treats "found" as update-in-place and "not found" +// as create, so a silent error would produce duplicate knowledge rows. func (r *knowledgeRepository) FindByMetadataKey( ctx context.Context, tenantID uint64, @@ -589,9 +597,16 @@ func (r *knowledgeRepository) FindByMetadataKey( value string, ) (*types.Knowledge, error) { var knowledge types.Knowledge - err := r.db.WithContext(ctx). + dialectName := r.db.Dialector.Name() + jsonPathExpr, err := database.JSONPathExpr(dialectName, "metadata", key) + if err != nil { + // Invalid metadata key (caller bug) - surface as an error rather + // than querying a different JSON path than intended. + return nil, fmt.Errorf("invalid metadata key %q: %w", key, err) + } + err = r.db.WithContext(ctx). Where("tenant_id = ? AND knowledge_base_id = ? AND deleted_at IS NULL", tenantID, kbID). - Where("metadata->>? = ?", key, value). + Where(jsonPathExpr+" = ?", value). First(&knowledge).Error if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { @@ -613,21 +628,28 @@ func (r *knowledgeRepository) FindByMetadataKeyPrefix( ) ([]*types.Knowledge, error) { escaped := escapeLikeKeyword(prefix) var items []*types.Knowledge - // The JSON key is embedded as a SQL literal (metadata->>'external_id'), NOT a - // bind parameter. PostgreSQL only uses the expression index - // idx_knowledges_kb_metadata_external_id (built on the literal expression - // (metadata->>'external_id')) when that exact expression appears in the query; - // a bound metadata->>$1 is a structurally different expression the planner - // cannot match, so it would silently fall back to a heap scan. key is an - // internal, caller-supplied field name (always "external_id"); single-quotes - // are doubled defensively so the literal is always well-formed. + // The JSON key is embedded as a SQL literal, NOT a bind parameter. PostgreSQL + // only uses the expression index idx_knowledges_kb_metadata_external_id (built + // on the literal expression (metadata->>'external_id')) when that exact + // expression appears in the query; a bound metadata->>$1 is a structurally + // different expression the planner cannot match, so it would silently fall + // back to a heap scan. MySQL indexes the equivalent generated column + // metadata_external_id, which its optimizer matches against the same literal + // extraction expression. + // + // The extraction syntax itself must go through database.JSONPathExpr: MySQL + // requires a '$.key' path and rejects the bare-key PostgreSQL form with + // error 3143 as soon as any row holds non-null JSON. // // The prefix pattern stays a bind parameter: an unnamed prepared statement is // custom-planned with the actual value, so LIKE 'prefix%' still extracts the // prefix and drives the index. The explicit ESCAPE '\' keeps backslash-escaped - // wildcards (e.g. \_) literal on both PostgreSQL and SQLite. - keyExpr := "metadata->>'" + strings.ReplaceAll(key, "'", "''") + "'" - err := r.db.WithContext(ctx). + // wildcards (e.g. \_) literal across dialects. + keyExpr, err := database.JSONPathExprIndexed(r.db.Dialector.Name(), "metadata", key) + if err != nil { + return nil, fmt.Errorf("invalid metadata key %q: %w", key, err) + } + err = r.db.WithContext(ctx). Where("tenant_id = ? AND knowledge_base_id = ? AND deleted_at IS NULL", tenantID, kbID). Where(keyExpr+" LIKE ? ESCAPE ?", escaped+"%", `\`). Find(&items).Error diff --git a/internal/application/repository/knowledge_metadata_test.go b/internal/application/repository/knowledge_metadata_test.go new file mode 100644 index 0000000000..05b6cdc8d1 --- /dev/null +++ b/internal/application/repository/knowledge_metadata_test.go @@ -0,0 +1,76 @@ +package repository + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +// TestFindByMetadataKey_SQLite exercises the dialect-aware JSON path +// branch of FindByMetadataKey. SQLite stores JSON as TEXT and supports +// json_extract(metadata, '$.external_id'); the historical code used the +// PostgreSQL `metadata->>? = ?` syntax which fails on MySQL with +// "Invalid JSON path expression" and is not the SQLite idiom either. +func TestFindByMetadataKey_SQLite(t *testing.T) { + db := setupKnowledgeTestDB(t) + repo := NewKnowledgeRepository(db) + ctx := context.Background() + + tenantID := uint64(1) + kbID := uuid.NewString() + + // Insert two knowledge rows with metadata containing external_id and + // datasource_id keys, matching the shape the datasource service writes. + insertKnowledgeWithMetadata(t, db, tenantID, kbID, "ext-1", "doc-1") + insertKnowledgeWithMetadata(t, db, tenantID, kbID, "ext-2", "doc-2") + + t.Run("exact match returns the right row", func(t *testing.T) { + got, err := repo.FindByMetadataKey(ctx, tenantID, kbID, "external_id", "ext-1") + require.NoError(t, err) + require.NotNil(t, got) + require.Equal(t, "doc-1", got.Source) + }) + + t.Run("no match returns nil nil (not an error)", func(t *testing.T) { + got, err := repo.FindByMetadataKey(ctx, tenantID, kbID, "external_id", "ext-missing") + require.NoError(t, err) + require.Nil(t, got) + }) + + t.Run("different key works", func(t *testing.T) { + got, err := repo.FindByMetadataKey(ctx, tenantID, kbID, "datasource_id", "ds-1") + require.NoError(t, err) + require.NotNil(t, got) + require.Equal(t, "doc-1", got.Source) + }) + + t.Run("soft-deleted row is not returned", func(t *testing.T) { + // Soft-delete the first row by setting deleted_at. + require.NoError(t, db.Exec( + `UPDATE knowledges SET deleted_at = '2024-01-01 00:00:00' WHERE source = ?`, + "doc-1").Error) + got, err := repo.FindByMetadataKey(ctx, tenantID, kbID, "external_id", "ext-1") + require.NoError(t, err) + require.Nil(t, got) + }) +} + +// insertKnowledgeWithMetadata seeds a knowledges row whose metadata +// JSON contains external_id and datasource_id keys, matching the shape +// the datasource service writes during sync. Each row gets a unique +// datasource_id so key-lookup tests can assert on a specific row. +func insertKnowledgeWithMetadata(t *testing.T, db *gorm.DB, tenantID uint64, kbID, externalID, source string) { + t.Helper() + id := uuid.NewString() + // datasource_id is derived from source so the "different key works" + // subtest can look up ds-1 and know it should get the doc-1 row. + datasourceID := "ds-" + source[len(source)-1:] + metadata := `{"external_id":"` + externalID + `","datasource_id":"` + datasourceID + `","source_resource_id":"rs-1"}` + require.NoError(t, db.Exec(` + INSERT INTO knowledges (id, tenant_id, knowledge_base_id, type, title, source, parse_status, metadata) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, id, tenantID, kbID, "file", source, source, "completed", metadata).Error) +} diff --git a/internal/application/repository/mcp_oauth_test.go b/internal/application/repository/mcp_oauth_test.go index cbada31684..ab05ed717d 100644 --- a/internal/application/repository/mcp_oauth_test.go +++ b/internal/application/repository/mcp_oauth_test.go @@ -122,3 +122,31 @@ func TestMCPOAuthRepositoryRefreshLeaseHasSingleOwner(t *testing.T) { require.NoError(t, err) require.True(t, second) } + +func TestMCPOAuthRepositoryTokenWithoutExpiryStoresNull(t *testing.T) { + repo := newMCPOAuthTestRepo(t) + ctx := context.Background() + principal := types.Principal{Type: types.PrincipalWebUser, ID: "u1"} + + require.NoError(t, repo.SaveTokenForPrincipal(ctx, &types.MCPOAuthToken{ + TenantID: 7, + UserID: principal.StorageID(), + PrincipalType: principal.Type, + PrincipalID: principal.ID, + ServiceID: "svc1", + AccessToken: "non-expiring-token", + TokenType: "Bearer", + })) + + var expiresAtIsNull bool + require.NoError(t, repo.db.Raw( + "SELECT expires_at IS NULL FROM mcp_oauth_tokens WHERE tenant_id = ? AND service_id = ?", + 7, + "svc1", + ).Scan(&expiresAtIsNull).Error) + require.True(t, expiresAtIsNull, "a provider token without an expiry must persist SQL NULL") + + token, err := repo.GetTokenForPrincipal(ctx, 7, principal, "svc1") + require.NoError(t, err) + require.True(t, token.ExpiresAt.IsZero()) +} diff --git a/internal/application/repository/message.go b/internal/application/repository/message.go index 22d83ac375..323ae0b496 100644 --- a/internal/application/repository/message.go +++ b/internal/application/repository/message.go @@ -7,6 +7,7 @@ import ( "gorm.io/gorm" + "github.com/Tencent/WeKnora/internal/database" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" ) @@ -152,7 +153,9 @@ func (r *messageRepository) GetMessageByRequestID( return &message, nil } -// SearchMessagesByKeyword searches messages by keyword (ILIKE) across sessions for a tenant +// SearchMessagesByKeyword searches messages by keyword (case-insensitive, +// dialect-aware via database.CaseInsensitiveLike: ILIKE on postgres, +// LOWER(col) LIKE LOWER(?) on mysql/sqlite) across sessions for a tenant func (r *messageRepository) SearchMessagesByKeyword( ctx context.Context, tenantID uint64, keyword string, sessionIDs []string, limit int, ) ([]*types.MessageWithSession, error) { @@ -168,7 +171,10 @@ func (r *messageRepository) SearchMessagesByKeyword( Joins("INNER JOIN sessions ON sessions.id = messages.session_id AND sessions.deleted_at IS NULL"). Where("sessions.tenant_id = ?", tenantID). Where("messages.deleted_at IS NULL"). - Where("messages.content ILIKE ?", "%"+escapeLikeKeyword(keyword)+"%") + Where( + database.CaseInsensitiveLike(r.db.Dialector.Name(), "messages.content", "?"), + "%"+escapeLikeKeyword(keyword)+"%", + ) if len(sessionIDs) > 0 { query = query.Where("messages.session_id IN ?", sessionIDs) diff --git a/internal/application/repository/model_usage.go b/internal/application/repository/model_usage.go index f8336870ad..9886e07350 100644 --- a/internal/application/repository/model_usage.go +++ b/internal/application/repository/model_usage.go @@ -7,7 +7,8 @@ import ( // scopeKnowledgeBasesByModelID filters knowledge_bases rows that reference // modelID in any model-binding field. func scopeKnowledgeBasesByModelID(db *gorm.DB, modelID string) *gorm.DB { - if db.Dialector.Name() == "postgres" { + switch db.Dialector.Name() { + case "postgres": return db.Where( "embedding_model_id = ? OR summary_model_id = ? OR "+ "image_processing_config->>'model_id' = ? OR "+ @@ -16,21 +17,32 @@ func scopeKnowledgeBasesByModelID(db *gorm.DB, modelID string) *gorm.DB { "wiki_config->>'synthesis_model_id' = ?", modelID, modelID, modelID, modelID, modelID, modelID, ) + case "mysql": + return db.Where( + "embedding_model_id = ? OR summary_model_id = ? OR "+ + "JSON_UNQUOTE(JSON_EXTRACT(image_processing_config, '$.model_id')) = ? OR "+ + "JSON_UNQUOTE(JSON_EXTRACT(vlm_config, '$.model_id')) = ? OR "+ + "JSON_UNQUOTE(JSON_EXTRACT(asr_config, '$.model_id')) = ? OR "+ + "JSON_UNQUOTE(JSON_EXTRACT(wiki_config, '$.synthesis_model_id')) = ?", + modelID, modelID, modelID, modelID, modelID, modelID, + ) + default: + return db.Where( + "embedding_model_id = ? OR summary_model_id = ? OR "+ + "json_extract(image_processing_config, '$.model_id') = ? OR "+ + "json_extract(vlm_config, '$.model_id') = ? OR "+ + "json_extract(asr_config, '$.model_id') = ? OR "+ + "json_extract(wiki_config, '$.synthesis_model_id') = ?", + modelID, modelID, modelID, modelID, modelID, modelID, + ) } - return db.Where( - "embedding_model_id = ? OR summary_model_id = ? OR "+ - "json_extract(image_processing_config, '$.model_id') = ? OR "+ - "json_extract(vlm_config, '$.model_id') = ? OR "+ - "json_extract(asr_config, '$.model_id') = ? OR "+ - "json_extract(wiki_config, '$.synthesis_model_id') = ?", - modelID, modelID, modelID, modelID, modelID, modelID, - ) } // scopeCustomAgentsByModelID filters custom_agents rows whose config JSON // references modelID in any model-binding field. func scopeCustomAgentsByModelID(db *gorm.DB, modelID string) *gorm.DB { - if db.Dialector.Name() == "postgres" { + switch db.Dialector.Name() { + case "postgres": return db.Where( "config->>'model_id' = ? OR config->>'rerank_model_id' = ? OR "+ "config->>'vlm_model_id' = ? OR config->>'asr_model_id' = ? OR "+ @@ -38,14 +50,25 @@ func scopeCustomAgentsByModelID(db *gorm.DB, modelID string) *gorm.DB { "config->'question_suggestions'->'follow_ups'->>'model_id' = ?", modelID, modelID, modelID, modelID, modelID, modelID, ) + case "mysql": + return db.Where( + "JSON_UNQUOTE(JSON_EXTRACT(config, '$.model_id')) = ? OR "+ + "JSON_UNQUOTE(JSON_EXTRACT(config, '$.rerank_model_id')) = ? OR "+ + "JSON_UNQUOTE(JSON_EXTRACT(config, '$.vlm_model_id')) = ? OR "+ + "JSON_UNQUOTE(JSON_EXTRACT(config, '$.asr_model_id')) = ? OR "+ + "JSON_UNQUOTE(JSON_EXTRACT(config, '$.query_understand_model_id')) = ? OR "+ + "JSON_UNQUOTE(JSON_EXTRACT(config, '$.question_suggestions.follow_ups.model_id')) = ?", + modelID, modelID, modelID, modelID, modelID, modelID, + ) + default: + return db.Where( + "json_extract(config, '$.model_id') = ? OR "+ + "json_extract(config, '$.rerank_model_id') = ? OR "+ + "json_extract(config, '$.vlm_model_id') = ? OR "+ + "json_extract(config, '$.asr_model_id') = ? OR "+ + "json_extract(config, '$.query_understand_model_id') = ? OR "+ + "json_extract(config, '$.question_suggestions.follow_ups.model_id') = ?", + modelID, modelID, modelID, modelID, modelID, modelID, + ) } - return db.Where( - "json_extract(config, '$.model_id') = ? OR "+ - "json_extract(config, '$.rerank_model_id') = ? OR "+ - "json_extract(config, '$.vlm_model_id') = ? OR "+ - "json_extract(config, '$.asr_model_id') = ? OR "+ - "json_extract(config, '$.query_understand_model_id') = ? OR "+ - "json_extract(config, '$.question_suggestions.follow_ups.model_id') = ?", - modelID, modelID, modelID, modelID, modelID, modelID, - ) } diff --git a/internal/application/repository/model_usage_test.go b/internal/application/repository/model_usage_test.go index a8358fef35..b6552aaee3 100644 --- a/internal/application/repository/model_usage_test.go +++ b/internal/application/repository/model_usage_test.go @@ -4,10 +4,12 @@ import ( "context" "testing" + "github.com/DATA-DOG/go-sqlmock" "github.com/Tencent/WeKnora/internal/types" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gorm.io/driver/mysql" "gorm.io/gorm" ) @@ -35,6 +37,56 @@ func setupModelUsageTestDB(t *testing.T) *gorm.DB { return db } +func setupModelUsageMySQLDryRunDB(t *testing.T) *gorm.DB { + t.Helper() + + sqlDB, _, err := sqlmock.New() + require.NoError(t, err) + t.Cleanup(func() { _ = sqlDB.Close() }) + + db, err := gorm.Open(mysql.New(mysql.Config{ + Conn: sqlDB, + SkipInitializeWithVersion: true, + }), &gorm.Config{DryRun: true}) + require.NoError(t, err) + return db +} + +func TestModelUsageScopesUnquoteMySQLJSONStringValues(t *testing.T) { + db := setupModelUsageMySQLDryRunDB(t) + modelID := "model-1" + + knowledgeBaseQuery := scopeKnowledgeBasesByModelID( + db.Model(&types.KnowledgeBase{}), + modelID, + ).Find(&[]types.KnowledgeBase{}) + knowledgeBaseSQL := knowledgeBaseQuery.Statement.SQL.String() + for _, expression := range []string{ + "JSON_UNQUOTE(JSON_EXTRACT(image_processing_config, '$.model_id')) = ?", + "JSON_UNQUOTE(JSON_EXTRACT(vlm_config, '$.model_id')) = ?", + "JSON_UNQUOTE(JSON_EXTRACT(asr_config, '$.model_id')) = ?", + "JSON_UNQUOTE(JSON_EXTRACT(wiki_config, '$.synthesis_model_id')) = ?", + } { + require.Contains(t, knowledgeBaseSQL, expression) + } + + customAgentQuery := scopeCustomAgentsByModelID( + db.Model(&types.CustomAgent{}), + modelID, + ).Find(&[]types.CustomAgent{}) + customAgentSQL := customAgentQuery.Statement.SQL.String() + for _, expression := range []string{ + "JSON_UNQUOTE(JSON_EXTRACT(config, '$.model_id')) = ?", + "JSON_UNQUOTE(JSON_EXTRACT(config, '$.rerank_model_id')) = ?", + "JSON_UNQUOTE(JSON_EXTRACT(config, '$.vlm_model_id')) = ?", + "JSON_UNQUOTE(JSON_EXTRACT(config, '$.asr_model_id')) = ?", + "JSON_UNQUOTE(JSON_EXTRACT(config, '$.query_understand_model_id')) = ?", + "JSON_UNQUOTE(JSON_EXTRACT(config, '$.question_suggestions.follow_ups.model_id')) = ?", + } { + require.Contains(t, customAgentSQL, expression) + } +} + func TestCountByModelID_KnowledgeBase(t *testing.T) { ctx := context.Background() db := setupModelUsageTestDB(t) diff --git a/internal/application/repository/mysql_integration_test.go b/internal/application/repository/mysql_integration_test.go new file mode 100644 index 0000000000..e385dcb1bc --- /dev/null +++ b/internal/application/repository/mysql_integration_test.go @@ -0,0 +1,396 @@ +package repository + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/mysql" + "gorm.io/gorm" + + "github.com/Tencent/WeKnora/internal/types" +) + +// The repository layer builds dialect-specific SQL, and the differences that +// matter most are the ones SQLite happens to tolerate. SQLite accepts the +// PostgreSQL bare-key form `metadata->>'external_id'`; MySQL rejects it with +// error 3143 as soon as a row holds non-null JSON. A suite that only ever runs +// on SQLite therefore reports green for queries that cannot run in production +// on MySQL at all, which is exactly how that shape of bug reaches users. +// +// These tests execute the real repositories against a real MySQL server using +// the schema from migrations/mysql. They are skipped unless +// WEKNORA_MYSQL_TEST_DSN points at a throwaway database; CI supplies one. +const mysqlTestDSNEnv = "WEKNORA_MYSQL_TEST_DSN" + +// mysqlBaselinePath is the consolidated MySQL schema. Tests load it directly +// rather than going through golang-migrate so that a schema mistake surfaces +// as a failing query rather than as a migration-tooling error. +const mysqlBaselinePath = "../../../migrations/mysql/000000_init.up.sql" + +func setupMySQLTestDB(t *testing.T) *gorm.DB { + t.Helper() + + dsn := strings.TrimSpace(os.Getenv(mysqlTestDSNEnv)) + if dsn == "" { + t.Skipf("%s is not set; skipping MySQL integration tests", mysqlTestDSNEnv) + } + + baseline, err := os.ReadFile(mysqlBaselinePath) + require.NoError(t, err, "read MySQL baseline schema") + + admin, err := gorm.Open(mysql.Open(dsn), &gorm.Config{}) + require.NoError(t, err, "connect to MySQL") + adminSQL, err := admin.DB() + require.NoError(t, err) + defer func() { _ = adminSQL.Close() }() + + // A dedicated database per test keeps runs independent and lets a failing + // test leave its rows behind for inspection without affecting the others. + schema := "weknora_it_" + strings.ReplaceAll(uuid.New().String(), "-", "")[:24] + require.NoError(t, admin.Exec("CREATE DATABASE `"+schema+"` CHARACTER SET utf8mb4").Error) + + db, err := gorm.Open(mysql.Open(mysqlDSNWithDatabase(t, dsn, schema)), &gorm.Config{}) + require.NoError(t, err) + sqlDB, err := db.DB() + require.NoError(t, err) + + for _, statement := range splitSQLStatements(string(baseline)) { + require.NoError(t, db.Exec(statement).Error, "apply baseline statement: %.120s", statement) + } + + t.Cleanup(func() { + _ = sqlDB.Close() + cleanup, err := gorm.Open(mysql.Open(dsn), &gorm.Config{}) + if err != nil { + return + } + defer func() { + if raw, err := cleanup.DB(); err == nil { + _ = raw.Close() + } + }() + _ = cleanup.Exec("DROP DATABASE IF EXISTS `" + schema + "`").Error + }) + + return db +} + +// mysqlDSNWithDatabase rewrites the database name in a go-sql-driver DSN, +// whose shape is user:pass@tcp(host:port)/dbname?params. +func mysqlDSNWithDatabase(t *testing.T, dsn, database string) string { + t.Helper() + slash := strings.LastIndex(dsn, "/") + require.Greater(t, slash, -1, "DSN %q must contain a / before the database name", dsn) + params := "" + if question := strings.Index(dsn[slash:], "?"); question > -1 { + params = dsn[slash+question:] + } + return dsn[:slash+1] + database + params +} + +// splitSQLStatements splits a migration file on statement terminators. Comments +// are dropped first: a `;` inside a comment would otherwise cut a CREATE TABLE +// in half and report the remainder as a syntax error. The baseline declares no +// stored programs, so a plain `;` split is sufficient once comments are gone. +func splitSQLStatements(script string) []string { + var statements []string + for _, chunk := range strings.Split(stripSQLComments(script), ";") { + if trimmed := strings.TrimSpace(chunk); trimmed != "" { + statements = append(statements, trimmed) + } + } + return statements +} + +func stripSQLComments(script string) string { + var kept []string + for _, line := range strings.Split(script, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "--") { + continue + } + kept = append(kept, line) + } + return strings.Join(kept, "\n") +} + +func insertMySQLKnowledge(t *testing.T, db *gorm.DB, tenantID uint64, kbID, externalID string) string { + t.Helper() + id := uuid.New().String() + require.NoError(t, db.Exec(` + INSERT INTO knowledges + (id, tenant_id, knowledge_base_id, type, title, source, parse_status, metadata) + VALUES (?, ?, ?, 'document', ?, 'feishu', 'completed', ?) + `, id, tenantID, kbID, externalID, fmt.Sprintf(`{"external_id":%q}`, externalID)).Error) + return id +} + +// insertMySQLTenant seeds a tenant so rows with a tenant_id foreign key can be +// inserted. tenants.id is AUTO_INCREMENT, so the assigned id is read back. +func insertMySQLTenant(t *testing.T, db *gorm.DB) uint64 { + t.Helper() + name := "it-" + uuid.New().String() + require.NoError(t, db.Exec( + `INSERT INTO tenants (name, business) VALUES (?, 'integration-test')`, name, + ).Error) + var id uint64 + require.NoError(t, db.Raw(`SELECT id FROM tenants WHERE name = ?`, name).Scan(&id).Error) + return id +} + +func insertMySQLChunk(t *testing.T, db *gorm.DB, tenantID uint64, kbID, knowledgeID, content string) string { + t.Helper() + id := uuid.New().String() + require.NoError(t, db.Exec(` + INSERT INTO chunks + (id, tenant_id, knowledge_id, knowledge_base_id, content, + chunk_index, start_at, end_at, is_enabled, chunk_type, flags, status) + VALUES (?, ?, ?, ?, ?, 0, 0, ?, TRUE, 'text', 0, 0) + `, id, tenantID, knowledgeID, kbID, content, len(content)).Error) + return id +} + +// TestMySQLFindByMetadataKeyPrefix is the regression test for the bare-key +// `metadata->>'external_id'` form. Its only caller logs and returns on error, +// so on MySQL the failure was silent: re-syncing a datasource stopped sweeping +// sub-items that had disappeared upstream, leaking orphan knowledge rows. +func TestMySQLFindByMetadataKeyPrefix(t *testing.T) { + db := setupMySQLTestDB(t) + repo := NewKnowledgeRepository(db).(*knowledgeRepository) + ctx := context.Background() + + const tenantID uint64 = 4242 + kbID := uuid.New().String() + otherKBID := uuid.New().String() + + _ = insertMySQLKnowledge(t, db, tenantID, kbID, "nodeA") + childID := insertMySQLKnowledge(t, db, tenantID, kbID, "nodeA#file#x") + _ = insertMySQLKnowledge(t, db, tenantID, kbID, "nodeB") + _ = insertMySQLKnowledge(t, db, tenantID, otherKBID, "nodeA#file#y") + + results, err := repo.FindByMetadataKeyPrefix(ctx, tenantID, kbID, "external_id", "nodeA#") + require.NoError(t, err) + require.Len(t, results, 1, "only the attachment child should match the prefix 'nodeA#'") + assert.Equal(t, childID, results[0].ID) +} + +func TestMySQLFindByMetadataKey(t *testing.T) { + db := setupMySQLTestDB(t) + repo := NewKnowledgeRepository(db) + ctx := context.Background() + + const tenantID uint64 = 4243 + kbID := uuid.New().String() + wanted := insertMySQLKnowledge(t, db, tenantID, kbID, "node-exact") + _ = insertMySQLKnowledge(t, db, tenantID, kbID, "node-other") + + found, err := repo.FindByMetadataKey(ctx, tenantID, kbID, "external_id", "node-exact") + require.NoError(t, err) + require.NotNil(t, found) + assert.Equal(t, wanted, found.ID) + + missing, err := repo.FindByMetadataKey(ctx, tenantID, kbID, "external_id", "absent") + require.NoError(t, err) + assert.Nil(t, missing) +} + +// TestMySQLMetadataExternalIDUsesIndex guards the generated column that +// materializes metadata->>'$.external_id'. Without it both lookups above are +// full scans, and a datasource sync runs one per item — invisible on the small +// datasets these tests use, painful on a real knowledge base. +func TestMySQLMetadataExternalIDUsesIndex(t *testing.T) { + db := setupMySQLTestDB(t) + + // possible_keys rather than key: which index the optimizer finally picks + // depends on table statistics, but possible_keys answers the question this + // test exists for — whether MySQL can match the predicate to the index at + // all. For the equality form that means generated-column substitution + // recognised metadata ->> '$.external_id'; MySQL never substitutes for LIKE, + // so the prefix form has to name the generated column itself. + for name, query := range map[string]string{ + "equality": "SELECT id FROM knowledges WHERE knowledge_base_id = 'kb' " + + "AND metadata ->> '$.external_id' = 'node'", + "prefix": "SELECT id FROM knowledges WHERE knowledge_base_id = 'kb' " + + "AND metadata_external_id LIKE 'node#%'", + } { + t.Run(name, func(t *testing.T) { + var plan struct { + PossibleKeys *string `gorm:"column:possible_keys"` + } + require.NoError(t, db.Raw("EXPLAIN "+query).Scan(&plan).Error) + require.NotNil(t, plan.PossibleKeys, "predicate must be indexable, got a full scan") + assert.Contains(t, *plan.PossibleKeys, "idx_knowledges_kb_metadata_external_id") + }) + } +} + +// TestMySQLUpdateChunks covers the batched CASE update. PostgreSQL needs the +// `(CASE ... END)::boolean` cast fed by string arguments, MySQL needs native +// bool/int under STRICT_TRANS_TABLES, and the two are easy to conflate: binding +// "true" to a tinyint raises error 1292, while casting on the PostgreSQL side +// with a native bool makes pgx refuse to encode the argument at all. +func TestMySQLUpdateChunks(t *testing.T) { + db := setupMySQLTestDB(t) + repo := NewChunkRepository(db) + ctx := context.Background() + + tenantID := insertMySQLTenant(t, db) + kbID := uuid.New().String() + knowledgeID := insertMySQLKnowledge(t, db, tenantID, kbID, "chunk-owner") + chunkID := insertMySQLChunk(t, db, tenantID, kbID, knowledgeID, "original content") + + updated := &types.Chunk{ + ID: chunkID, + TenantID: tenantID, + KnowledgeID: knowledgeID, + KnowledgeBaseID: kbID, + Content: "rewritten content", + IsEnabled: false, + Flags: 3, + Status: 2, + } + require.NoError(t, repo.UpdateChunks(ctx, []*types.Chunk{updated})) + + var stored struct { + Content string + IsEnabled bool + Flags int + Status int + } + require.NoError(t, db.Raw( + "SELECT content, is_enabled, flags, status FROM chunks WHERE id = ?", chunkID, + ).Scan(&stored).Error) + assert.Equal(t, "rewritten content", stored.Content) + assert.False(t, stored.IsEnabled) + assert.Equal(t, 3, stored.Flags) + assert.Equal(t, 2, stored.Status) +} + +// TestMySQLSeqIDStartsAboveReservedRange pins the AUTO_INCREMENT start values +// to the PostgreSQL sequence start values. FAQ import lets a caller choose a +// seq_id below the start value (types.FAQImportEntry.ID documents the rule), so +// generated values entering that range collide with imported ones. +func TestMySQLSeqIDStartsAboveReservedRange(t *testing.T) { + db := setupMySQLTestDB(t) + + tenantID := insertMySQLTenant(t, db) + kbID := uuid.New().String() + knowledgeID := insertMySQLKnowledge(t, db, tenantID, kbID, "seq-owner") + chunkID := insertMySQLChunk(t, db, tenantID, kbID, knowledgeID, "seq probe") + + var chunkSeqID int64 + require.NoError(t, db.Raw("SELECT seq_id FROM chunks WHERE id = ?", chunkID).Scan(&chunkSeqID).Error) + assert.GreaterOrEqual(t, chunkSeqID, int64(100000000), + "generated chunk seq_id must stay out of the range reserved for FAQ import") + + tagID := uuid.New().String() + require.NoError(t, db.Exec(` + INSERT INTO knowledge_tags (id, tenant_id, knowledge_base_id, name) + VALUES (?, ?, ?, 'seq probe tag') + `, tagID, tenantID, kbID).Error) + + var tagSeqID int64 + require.NoError(t, db.Raw("SELECT seq_id FROM knowledge_tags WHERE id = ?", tagID).Scan(&tagSeqID).Error) + assert.GreaterOrEqual(t, tagSeqID, int64(10000000)) +} + +// TestMySQLSessionDefaultsMatchPostgres pins the seeded fallback answer. Its +// value is non-ASCII, so it also catches the schema being applied over a +// connection that is not utf8mb4: MySQL records the DDL-time character set in +// an expression default, and a latin1 connection stores mojibake that no query +// ever complains about. +func TestMySQLSessionDefaultsMatchPostgres(t *testing.T) { + db := setupMySQLTestDB(t) + + tenantID := insertMySQLTenant(t, db) + sessionID := uuid.New().String() + require.NoError(t, db.Exec( + `INSERT INTO sessions (id, tenant_id) VALUES (?, ?)`, sessionID, tenantID, + ).Error) + + var fallback string + require.NoError(t, db.Raw( + `SELECT fallback_response FROM sessions WHERE id = ?`, sessionID, + ).Scan(&fallback).Error) + assert.Equal(t, "很抱歉，我暂时无法回答这个问题。", fallback) +} + +// TestMySQLCaseInsensitiveSearches covers the ILIKE replacements. Each of these +// is a plain syntax error on MySQL if the dialect helper is bypassed. +func TestMySQLCaseInsensitiveSearches(t *testing.T) { + db := setupMySQLTestDB(t) + ctx := context.Background() + + tenantID := insertMySQLTenant(t, db) + + userID := uuid.New().String() + require.NoError(t, db.Exec(` + INSERT INTO users (id, username, email, password_hash, tenant_id, is_active) + VALUES (?, 'AliceExample', 'Alice@Example.COM', 'hash', ?, TRUE) + `, userID, tenantID).Error) + + users, err := NewUserRepository(db).SearchUsers(ctx, "alice", 10) + require.NoError(t, err) + require.Len(t, users, 1) + assert.Equal(t, userID, users[0].ID) + + sessionID := uuid.New().String() + require.NoError(t, db.Exec(` + INSERT INTO sessions (id, tenant_id, title, knowledge_base_id) + VALUES (?, ?, 'Quarterly Planning', ?) + `, sessionID, tenantID, uuid.New().String()).Error) + + messageID := uuid.New().String() + require.NoError(t, db.Exec(` + INSERT INTO messages (id, session_id, request_id, role, content) + VALUES (?, ?, ?, 'user', 'Please Summarize The Roadmap') + `, messageID, sessionID, uuid.New().String()).Error) + + messages, err := NewMessageRepository(db). + SearchMessagesByKeyword(ctx, tenantID, "summarize the", nil, 10) + require.NoError(t, err) + require.Len(t, messages, 1) + assert.Equal(t, messageID, messages[0].ID) +} + +// TestMySQLTaskQueueClaimAndFail exercises FOR UPDATE SKIP LOCKED (MySQL 8 +// supports it, so the dialect gate must open rather than fall through to the +// single-writer SQLite path) and the UPDATE ... RETURNING replacement. +func TestMySQLTaskQueueClaimAndFail(t *testing.T) { + db := setupMySQLTestDB(t) + repo := NewTaskPendingOpsRepository(db) + ctx := context.Background() + + kbID := uuid.New().String() + for index := 0; index < 3; index++ { + require.NoError(t, repo.Enqueue(ctx, &types.TaskPendingOp{ + TenantID: 4247, + TaskType: types.TypeWikiIngest, + Scope: types.TaskScopeKnowledgeBase, + ScopeID: kbID, + Op: "ingest", + DedupKey: fmt.Sprintf("knowledge-%d", index), + Payload: []byte(`{}`), + })) + } + + claimed, err := repo.ClaimBatch( + ctx, types.TypeWikiIngest, types.TaskScopeKnowledgeBase, kbID, 10, time.Now().UTC(), + ) + require.NoError(t, err) + require.Len(t, claimed, 3) + + count, err := repo.IncrFailCount(ctx, claimed[0].ID) + require.NoError(t, err) + assert.Equal(t, 1, count) + + missing, err := repo.IncrFailCount(ctx, claimed[0].ID+100000) + require.NoError(t, err, "a missing row must not be an error, the caller treats 0 as already-gone") + assert.Equal(t, 0, missing) +} diff --git a/internal/application/repository/organization.go b/internal/application/repository/organization.go index 8aea067654..c9da94426e 100644 --- a/internal/application/repository/organization.go +++ b/internal/application/repository/organization.go @@ -5,6 +5,7 @@ import ( "errors" "time" + "github.com/Tencent/WeKnora/internal/database" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "gorm.io/gorm" @@ -90,8 +91,17 @@ func (r *organizationRepository) ListSearchable(ctx context.Context, query strin q := r.db.WithContext(ctx).Where("searchable = ?", true) if query != "" { pattern := "%" + query + "%" - // 支持按名称、描述或空间 ID 搜索，便于区分同名空间 - q = q.Where("name ILIKE ? OR description ILIKE ? OR id::text ILIKE ?", pattern, pattern, pattern) + // 支持按名称、描述或空间 ID 搜索，便于区分同名空间。 + // id 是 varchar(36)，本身就是文本类型，PG 上原来的 id::text + // 是冗余的；去掉后查询在 mysql/sqlite 上也直接可用。 + // CaseInsensitiveLike 保留 PG 的 ILIKE（命中 pg_trgm 索引）， + // 其余方言用 LOWER() LIKE LOWER()。 + dialect := r.db.Dialector.Name() + q = q.Where( + database.CaseInsensitiveLike(dialect, "name", "?")+" OR "+ + database.CaseInsensitiveLike(dialect, "description", "?")+" OR "+ + database.CaseInsensitiveLike(dialect, "id", "?"), + pattern, pattern, pattern) } err := q.Order("created_at DESC").Limit(limit).Find(&orgs).Error if err != nil { diff --git a/internal/application/repository/session.go b/internal/application/repository/session.go index 82ad99d90e..bf147a7b18 100644 --- a/internal/application/repository/session.go +++ b/internal/application/repository/session.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/Tencent/WeKnora/internal/database" apperrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -144,16 +145,18 @@ func (r *sessionRepository) QueryPaged( ) ([]*types.SessionListItem, int64, error) { // Dialect-aware bits so the same query works on Postgres and SQLite (Lite build). isPostgres := r.db.Dialector.Name() == "postgres" - titleLikeExpr := "LOWER(s.title) LIKE LOWER(?)" - if isPostgres { - titleLikeExpr = "s.title ILIKE ?" - } - // SQLite (the driver used by Lite) does not support NULLS LAST; its default - // nulls ordering puts NULLs first for DESC, which is actually what we want - // for pinned_at (rows with pinned_at=NULL are never pinned, so they get - // filtered to the tail by the preceding is_pinned DESC anyway). + isMySQL := r.db.Dialector.Name() == "mysql" + // CaseInsensitiveLike keeps ILIKE on Postgres (where pg_trgm GIN + // indexes can serve it) and emits LOWER() LIKE LOWER() elsewhere. + titleLikeExpr := database.CaseInsensitiveLike(r.db.Dialector.Name(), "s.title", "?") + // PostgreSQL supports NULLS LAST natively. MySQL does not, so emulate + // it with a CASE expression (NULL pinned_at sorts last). SQLite's + // default nulls ordering for DESC already puts NULLs first, which is + // acceptable for pinned_at (NULL = never pinned, filtered by is_pinned). orderClause := "s.is_pinned DESC, s.pinned_at DESC NULLS LAST, s.updated_at DESC" - if !isPostgres { + if isMySQL { + orderClause = "s.is_pinned DESC, CASE WHEN s.pinned_at IS NULL THEN 1 ELSE 0 END, s.pinned_at DESC, s.updated_at DESC" + } else if !isPostgres { orderClause = "s.is_pinned DESC, s.pinned_at DESC, s.updated_at DESC" } diff --git a/internal/application/repository/system_setting.go b/internal/application/repository/system_setting.go index bfe05755da..081108b869 100644 --- a/internal/application/repository/system_setting.go +++ b/internal/application/repository/system_setting.go @@ -29,8 +29,14 @@ func NewSystemSettingRepository(db *gorm.DB) interfaces.SystemSettingRepository // to ENV / default", so a 404 here is a normal control-flow signal, // not an error. Real DB errors (connection lost, etc.) surface up. func (r *systemSettingRepository) Get(ctx context.Context, key string) (*types.SystemSetting, error) { + if key == "" { + return nil, nil + } var s types.SystemSetting - err := r.db.WithContext(ctx).Where("key = ?", key).First(&s).Error + // Use a struct condition rather than a raw "key = ?" string: GORM + // quotes the column name per dialect, which matters because `key` + // is a reserved word in MySQL. On PostgreSQL the quoting is a no-op. + err := r.db.WithContext(ctx).Where(&types.SystemSetting{Key: key}).First(&s).Error if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return nil, nil @@ -44,7 +50,16 @@ func (r *systemSettingRepository) Get(ctx context.Context, key string) (*types.S // for stable management-UI rendering. No pagination — see type comment. func (r *systemSettingRepository) List(ctx context.Context) ([]*types.SystemSetting, error) { var rows []*types.SystemSetting - err := r.db.WithContext(ctx).Order("category ASC, key ASC").Find(&rows).Error + // Use clause.OrderBy with OrderByColumn so GORM quotes each column + // name per dialect. `key` is a reserved word in MySQL; a raw + // "key ASC" string would fail there because GORM does not re-parse + // raw Order strings. + err := r.db.WithContext(ctx).Clauses(clause.OrderBy{ + Columns: []clause.OrderByColumn{ + {Column: clause.Column{Name: "category"}}, + {Column: clause.Column{Name: "key"}}, + }, + }).Find(&rows).Error if err != nil { return nil, err } @@ -82,7 +97,11 @@ func (r *systemSettingRepository) Upsert(ctx context.Context, s *types.SystemSet // rather than translating to gorm.ErrRecordNotFound so the caller's // happy path is a single nil check on err. func (r *systemSettingRepository) Delete(ctx context.Context, key string) (bool, error) { - res := r.db.WithContext(ctx).Where("key = ?", key).Delete(&types.SystemSetting{}) + if key == "" { + return false, nil + } + // Struct condition so GORM quotes the reserved-word column `key` per dialect. + res := r.db.WithContext(ctx).Where(&types.SystemSetting{Key: key}).Delete(&types.SystemSetting{}) if res.Error != nil { return false, res.Error } diff --git a/internal/application/repository/task_queue.go b/internal/application/repository/task_queue.go index 63df641137..6d9bd76838 100644 --- a/internal/application/repository/task_queue.go +++ b/internal/application/repository/task_queue.go @@ -13,6 +13,15 @@ import ( "gorm.io/gorm/clause" ) +// dialectSupportsSkipLocked reports whether a GORM dialector with the +// given name supports the FOR UPDATE SKIP LOCKED syntax. PostgreSQL +// (all supported versions) and MySQL 8.0+ both do, with identical +// syntax. SQLite does not — the claim path falls back to a non-locking +// SELECT there. +func dialectSupportsSkipLocked(dialectName string) bool { + return dialectName == "postgres" || dialectName == "mysql" +} + // taskPendingOpsRepository implements interfaces.TaskPendingOpsRepository. type taskPendingOpsRepository struct { db *gorm.DB @@ -25,7 +34,7 @@ func NewTaskPendingOpsRepository(db *gorm.DB) interfaces.TaskPendingOpsRepositor // Enqueue inserts a single op. Callers must populate TenantID/TaskType/ // Scope/ScopeID/Op (Payload optional). ID, FailCount default to zero; -// EnqueuedAt is filled with the DB-side default if left zero. +// EnqueuedAt is filled with the current UTC time if left zero. func (r *taskPendingOpsRepository) Enqueue(ctx context.Context, op *types.TaskPendingOp) error { if err := preparePendingOp(op); err != nil { return err @@ -49,15 +58,21 @@ func preparePendingOp(op *types.TaskPendingOp) error { // driver-level default handling. op.Payload = []byte("{}") } + if op.EnqueuedAt.IsZero() { + // GORM sends Go's zero time explicitly on some create paths. MySQL + // rejects that value under STRICT_TRANS_TABLES instead of applying + // the column's CURRENT_TIMESTAMP default. + op.EnqueuedAt = time.Now().UTC() + } return nil } // EnqueueIfKnowledgeBaseActive prevents detached wiki cleanup from writing new -// durable work after a KB was soft-deleted. On Postgres the share lock -// serializes this check+insert transaction against the row update performed by -// soft deletion: whichever operation acquires the row first determines the -// order, and the deletion path's subsequent scope scrub removes any insert -// that committed before it. +// durable work after a KB was soft-deleted. On PostgreSQL and MySQL the share +// lock serializes this check+insert transaction against the row update +// performed by soft deletion: whichever operation acquires the row first +// determines the order, and the deletion path's subsequent scope scrub removes +// any insert that committed before it. func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive( ctx context.Context, op *types.TaskPendingOp, @@ -74,7 +89,7 @@ func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive( Select("id"). Where("id = ? AND tenant_id = ?", op.ScopeID, op.TenantID) dialector := tx.Dialector - if dialector.Name() == "postgres" { + if dialector.Name() == "postgres" || dialector.Name() == "mysql" { query = query.Clauses(clause.Locking{Strength: "SHARE"}) } var kb types.KnowledgeBase @@ -206,13 +221,13 @@ func (r *taskPendingOpsRepository) PeekBatch( // (claimed_at < staleBefore), AND the key has no fresh claim. The whole thing // runs in one transaction: // -// - Postgres: we lock the ANCHOR row (earliest eligible id) of each +// - PostgreSQL/MySQL: we lock the ANCHOR row (earliest eligible id) of each // candidate dedup_key with FOR UPDATE SKIP LOCKED. Because the anchor // uniquely represents its key, SKIP LOCKED hands concurrent claimers // DISJOINT key sets — a key whose anchor is already locked by another // in-flight claim is skipped entirely rather than half-claimed. We then // stamp every eligible row of the chosen keys and read them back. -// - Other dialects (SQLite, used by unit tests / Lite mode): writes are +// - SQLite (used by unit tests / Lite mode): writes are // serialized by the single-writer engine, so a plain grouped SELECT + // UPDATE is already race-free. // @@ -234,15 +249,22 @@ func (r *taskPendingOpsRepository) ClaimBatch( now := time.Now() var claimed []*types.TaskPendingOp err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + // Isolation: GORM Transaction() uses the DB's default isolation level. + // PG defaults to READ COMMITTED; MySQL defaults to REPEATABLE READ. + // REPEATABLE READ is STRICTER (more locking) but preserves correctness — + // concurrent claimers still get disjoint key sets via SKIP LOCKED. The + // trade-off is potentially higher lock contention on MySQL under load. + // 1. Pick up to `limit` distinct dedup_keys to claim, oldest first. // Keys with a fresh claim are excluded WHOLESALE so a late sibling // of an in-flight document never gets claimed on its own. var keys []string - if tx.Dialector.Name() == "postgres" { + if dialectSupportsSkipLocked(tx.Dialector.Name()) { // Lock the anchor (earliest eligible) row of each key with SKIP // LOCKED so concurrent claimers get disjoint KEY sets, then map // the locked anchors back to their dedup_keys. The NOT IN subquery // drops any key that still has a fresh (non-stale) claim. + // PostgreSQL and MySQL 8.0+ both support this syntax. const anchorSQL = ` SELECT dedup_key FROM task_pending_ops WHERE id IN ( @@ -354,18 +376,32 @@ func (r *taskPendingOpsRepository) DeleteByScope(ctx context.Context, scope, sco } // IncrFailCount atomically bumps fail_count for one row and returns the -// new value. We use UPDATE ... RETURNING so the read+write happens in -// one round trip and races between concurrent IncrFailCount callers -// resolve to monotonic counts. -// -// A missing row returns (0, nil): the caller's ID may have been removed -// by a concurrent DeleteByIDs (e.g. dead-letter path), which is benign. +// new value. Postgres uses UPDATE ... RETURNING (single round trip); +// MySQL has no RETURNING clause, so it uses an explicit transaction +// (UPDATE + SELECT on the same tx). A missing row returns (0, nil). func (r *taskPendingOpsRepository) IncrFailCount(ctx context.Context, id int64) (int, error) { + if r.db.Dialector.Name() == "postgres" { + var newCount int + err := r.db.WithContext(ctx).Raw( + "UPDATE task_pending_ops SET fail_count = fail_count + 1 WHERE id = ? RETURNING fail_count", id, + ).Scan(&newCount).Error + if err != nil { + return 0, err + } + return newCount, nil + } var newCount int - err := r.db.WithContext(ctx).Raw( - `UPDATE task_pending_ops SET fail_count = fail_count + 1 WHERE id = ? RETURNING fail_count`, - id, - ).Scan(&newCount).Error + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if err := tx.Model(&types.TaskPendingOp{}). + Where("id = ?", id). + UpdateColumn("fail_count", gorm.Expr("fail_count + 1")).Error; err != nil { + return err + } + return tx.Model(&types.TaskPendingOp{}). + Where("id = ?", id). + Select("fail_count"). + Scan(&newCount).Error + }) if err != nil { return 0, err } @@ -441,6 +477,13 @@ func (r *taskDeadLetterRepository) Insert(ctx context.Context, dl *types.TaskDea if len(dl.Payload) == 0 { dl.Payload = []byte("{}") } + if dl.FailedAt.IsZero() { + // FailedAt is not one of GORM's conventional auto-timestamp fields. + // Without an explicit value GORM sends Go's zero time, which MySQL + // rejects under STRICT_TRANS_TABLES instead of applying the column's + // CURRENT_TIMESTAMP default. + dl.FailedAt = time.Now().UTC() + } return r.db.WithContext(ctx).Create(dl).Error } diff --git a/internal/application/repository/task_queue_test.go b/internal/application/repository/task_queue_test.go index e69bf9d6bb..7c57460c76 100644 --- a/internal/application/repository/task_queue_test.go +++ b/internal/application/repository/task_queue_test.go @@ -189,8 +189,8 @@ func TestTaskPendingOps_SeedKnowledgeFinalizingSkipsDeletedKnowledgeBase(t *test // ---------------- TaskPendingOpsRepository ---------------- // TestTaskPendingOps_Enqueue_AssignsIDAndDefaults verifies a freshly -// inserted op gets a positive ID and the empty payload becomes "{}" -// rather than NULL/empty. +// inserted op gets a positive ID, a concrete UTC enqueue timestamp, and an +// empty payload becomes "{}" rather than NULL/empty. func TestTaskPendingOps_Enqueue_AssignsIDAndDefaults(t *testing.T) { db := setupTaskQueueTestDB(t) repo := NewTaskPendingOpsRepository(db) @@ -199,6 +199,8 @@ func TestTaskPendingOps_Enqueue_AssignsIDAndDefaults(t *testing.T) { op := makePendingOp("wiki:ingest", "knowledge_base", "kb-1", "ingest", "k-1", nil) require.NoError(t, repo.Enqueue(ctx, op)) assert.NotZero(t, op.ID) + assert.False(t, op.EnqueuedAt.IsZero()) + assert.Equal(t, time.UTC, op.EnqueuedAt.Location()) assert.Equal(t, json.RawMessage("{}"), op.Payload, "nil payload should default to {}") } @@ -427,6 +429,25 @@ func TestTaskPendingOps_IncrFailCount_ReturnsNewValueAndPersists(t *testing.T) { assert.Equal(t, 2, rows[0].FailCount) } +// TestTaskPendingOps_IncrFailCount_MissingRowReturnsZeroNil locks in the +// contract documented on IncrFailCount: a non-existent ID returns +// (0, nil) rather than an error. The caller's ID may have been removed +// by a concurrent DeleteByIDs (e.g. dead-letter path), which is benign. +// +// This test is added ahead of the RETURNING → UPDATE+SELECT rewrite +// for MySQL compatibility: the rewrite must preserve this exact +// behaviour across all dialects. +func TestTaskPendingOps_IncrFailCount_MissingRowReturnsZeroNil(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskPendingOpsRepository(db) + ctx := context.Background() + + // No row with this ID was ever inserted. + n, err := repo.IncrFailCount(ctx, 99999) + require.NoError(t, err, "missing row must not surface as an error") + assert.Equal(t, 0, n, "missing row must return count 0") +} + // TestTaskPendingOps_PendingCount_ScopedTuple confirms the count covers // only the (task_type, scope, scope_id) tuple. func TestTaskPendingOps_PendingCount_ScopedTuple(t *testing.T) { @@ -700,8 +721,8 @@ func makeDeadLetter(taskType, scope, scopeID, relatedID, lastErr string) *types. } } -// TestTaskDeadLetter_Insert_DefaultsAndAssignsID covers the empty-payload -// fallback and ID assignment. +// TestTaskDeadLetter_Insert_DefaultsAndAssignsID covers repository-side +// defaults for fields that GORM would otherwise send as explicit zero values. func TestTaskDeadLetter_Insert_DefaultsAndAssignsID(t *testing.T) { db := setupTaskQueueTestDB(t) repo := NewTaskDeadLetterRepository(db) @@ -714,11 +735,14 @@ func TestTaskDeadLetter_Insert_DefaultsAndAssignsID(t *testing.T) { FailCount: 3, // Scope intentionally empty — should default to "unknown". // Payload intentionally nil — should default to "{}". + // FailedAt intentionally zero — should become a concrete UTC time. } require.NoError(t, repo.Insert(ctx, dl)) assert.NotZero(t, dl.ID) assert.Equal(t, types.TaskScopeUnknown, dl.Scope) assert.Equal(t, json.RawMessage("{}"), dl.Payload) + assert.False(t, dl.FailedAt.IsZero()) + assert.Equal(t, time.UTC, dl.FailedAt.Location()) } // TestTaskDeadLetter_Insert_RejectsMissingFields verifies the guard @@ -835,3 +859,31 @@ func TestTaskDeadLetter_DeleteByID_IsIdempotent(t *testing.T) { require.NoError(t, err) assert.Len(t, rows, 0) } + +// ---------------- dialect capability gating ---------------- + +// TestDialectSupportsSkipLocked verifies the row-locking capability gate +// used by ClaimBatch. PostgreSQL and MySQL 8.0+ (the only MySQL versions +// WeKnora accepts) both support FOR UPDATE SKIP LOCKED with identical +// syntax; SQLite and others fall back to a non-locking SELECT. +func TestDialectSupportsSkipLocked(t *testing.T) { + tests := []struct { + dialect string + want bool + }{ + {"postgres", true}, + {"mysql", true}, + {"sqlite", false}, + {"", false}, + {"sqlserver", false}, + } + + for _, tt := range tests { + t.Run(tt.dialect, func(t *testing.T) { + got := dialectSupportsSkipLocked(tt.dialect) + if got != tt.want { + t.Fatalf("dialectSupportsSkipLocked(%q) = %v; want %v", tt.dialect, got, tt.want) + } + }) + } +} diff --git a/internal/application/repository/user.go b/internal/application/repository/user.go index b52740c60b..39c91958db 100644 --- a/internal/application/repository/user.go +++ b/internal/application/repository/user.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "github.com/Tencent/WeKnora/internal/database" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "gorm.io/gorm" @@ -261,8 +262,12 @@ func (r *userRepository) SearchUsers(ctx context.Context, query string, limit in var users []*types.User searchPattern := "%" + query + "%" + // CaseInsensitiveLike keeps ILIKE on Postgres (pg_trgm indexes) and + // falls back to LOWER() LIKE LOWER() on mysql/sqlite. + dialect := r.db.Dialector.Name() dbQuery := r.db.WithContext(ctx). - Where("username ILIKE ? OR email ILIKE ?", searchPattern, searchPattern). + Where(database.CaseInsensitiveLike(dialect, "username", "?")+" OR "+ + database.CaseInsensitiveLike(dialect, "email", "?"), searchPattern, searchPattern). Where("is_active = ?", true). Order("username ASC") diff --git a/internal/application/repository/wiki_dialect.go b/internal/application/repository/wiki_dialect.go new file mode 100644 index 0000000000..fec051ffd5 --- /dev/null +++ b/internal/application/repository/wiki_dialect.go @@ -0,0 +1,125 @@ +package repository + +import ( + "fmt" + + "github.com/Tencent/WeKnora/internal/database" + "gorm.io/gorm" +) + +// Dialect-aware SQL fragments for wiki_page.go. Each helper returns a +// per-dialect SQL expression; PG keeps its native operators, MySQL/SQLite +// get portable equivalents. Where PG has no MySQL analogue (trigram +// similarity, full-text search), the fallback uses LIKE with different +// matching and ranking semantics. + +func wikiJSONArrayLength(dialectName, column string) string { + switch dialectName { + case "postgres": + return fmt.Sprintf("COALESCE(jsonb_array_length(%s), 0)", column) + case "mysql": + return fmt.Sprintf("COALESCE(JSON_LENGTH(%s), 0)", column) + default: + return fmt.Sprintf("COALESCE(json_array_length(%s), 0)", column) + } +} + +// wikiJSONContains tests JSON array containment. SQLite uses json_each; +// callers must use wikiJSONContainsArg to shape the bind value. +func wikiJSONContains(dialectName, column string) string { + switch dialectName { + case "postgres": + return fmt.Sprintf("%s @> ?::jsonb", column) + case "mysql": + return fmt.Sprintf("JSON_CONTAINS(%s, ?)", column) + default: + return fmt.Sprintf("EXISTS (SELECT 1 FROM json_each(%s) WHERE value = ?)", column) + } +} + +// wikiJSONContainsArg: PG/MySQL get the JSON-encoded array; SQLite gets +// the bare scalar (json_each compares with =). +func wikiJSONContainsArg(dialectName, needleJSON, scalarValue string) string { + if dialectName == "sqlite" { + return scalarValue + } + return needleJSON +} + +func wikiJSONAsText(dialectName, column string) string { + switch dialectName { + case "postgres": + return fmt.Sprintf("%s::text", column) + case "mysql": + return fmt.Sprintf("CAST(%s AS CHAR)", column) + default: + return fmt.Sprintf("CAST(%s AS TEXT)", column) + } +} + +func wikiJSONEqual(dialectName, column string) string { + switch dialectName { + case "postgres": + return fmt.Sprintf("%s::jsonb = ?::jsonb", column) + case "mysql": + return fmt.Sprintf("%s = CAST(? AS JSON)", column) + default: + return fmt.Sprintf("%s = ?", column) + } +} + +// wikiCaseInsensitiveRegex: PG ~*, MySQL REGEXP_LIKE('i'), SQLite LIKE. +func wikiCaseInsensitiveRegex(dialectName, column, placeholder string) string { + switch dialectName { + case "postgres": + return fmt.Sprintf("%s ~* %s", column, placeholder) + case "mysql": + return fmt.Sprintf("REGEXP_LIKE(%s, %s, 'i')", column, placeholder) + default: + return fmt.Sprintf("%s LIKE %s", column, placeholder) + } +} + +// wikiSimilarityRank: PG trigram similarity; MySQL/SQLite degrade to +// a binary "contains" match (1/0). +func wikiSimilarityRank(dialectName, column, placeholder string) string { + switch dialectName { + case "postgres": + return fmt.Sprintf("similarity(lower(%s), %s)", column, placeholder) + case "mysql": + return fmt.Sprintf("(CASE WHEN LOWER(%s) LIKE CONCAT('%%', LOWER(%s), '%%') THEN 1 ELSE 0 END)", column, placeholder) + default: + return fmt.Sprintf("(CASE WHEN LOWER(%s) LIKE '%%' || LOWER(%s) || '%%' THEN 1 ELSE 0 END)", column, placeholder) + } +} + +func wikiSimilarityThreshold(dialectName, column, placeholder string) string { + switch dialectName { + case "postgres": + return fmt.Sprintf("lower(%s) %% %s", column, placeholder) + case "mysql": + return fmt.Sprintf("LOWER(%s) LIKE CONCAT('%%', LOWER(%s), '%%')", column, placeholder) + default: + return fmt.Sprintf("LOWER(%s) LIKE '%%' || LOWER(%s) || '%%'", column, placeholder) + } +} + +// wikiFullTextSearch: PG to_tsvector; MySQL/SQLite use multi-column LIKE. +func wikiFullTextSearch(dialectName string) string { + switch dialectName { + case "postgres": + return "(to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(content, '')) " + + "@@ plainto_tsquery('simple', ?) OR aliases::text ILIKE ?)" + default: + return "(" + database.CaseInsensitiveLike(dialectName, "title", "?") + + " OR " + database.CaseInsensitiveLike(dialectName, "content", "?") + + " OR " + database.CaseInsensitiveLike(dialectName, "aliases", "?") + ")" + } +} + +func wikiDialectName(db *gorm.DB) string { + if db == nil || db.Dialector == nil { + return "" + } + return db.Dialector.Name() +} diff --git a/internal/application/repository/wiki_dialect_test.go b/internal/application/repository/wiki_dialect_test.go new file mode 100644 index 0000000000..2f46cebe8e --- /dev/null +++ b/internal/application/repository/wiki_dialect_test.go @@ -0,0 +1,110 @@ +package repository + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestWikiDialectHelpers covers the SQL fragment shape for each +// dialect-aware wiki helper. These are pure-string functions, so the +// test asserts the exact output per dialect. + +func TestWikiJSONArrayLength(t *testing.T) { + tests := []struct { + dialect, want string + }{ + {"postgres", "COALESCE(jsonb_array_length(in_links), 0)"}, + {"mysql", "COALESCE(JSON_LENGTH(in_links), 0)"}, + {"sqlite", "COALESCE(json_array_length(in_links), 0)"}, + {"unknown", "COALESCE(json_array_length(in_links), 0)"}, + } + for _, tt := range tests { + t.Run(tt.dialect, func(t *testing.T) { + assert.Equal(t, tt.want, wikiJSONArrayLength(tt.dialect, "in_links")) + }) + } +} + +func TestWikiJSONContains(t *testing.T) { + tests := []struct { + dialect, want string + }{ + {"postgres", "source_refs @> ?::jsonb"}, + {"mysql", "JSON_CONTAINS(source_refs, ?)"}, + {"sqlite", "EXISTS (SELECT 1 FROM json_each(source_refs) WHERE value = ?)"}, + {"unknown", "EXISTS (SELECT 1 FROM json_each(source_refs) WHERE value = ?)"}, + } + for _, tt := range tests { + t.Run(tt.dialect, func(t *testing.T) { + assert.Equal(t, tt.want, wikiJSONContains(tt.dialect, "source_refs")) + }) + } +} + +func TestWikiJSONContainsArg(t *testing.T) { + // Postgres / MySQL take the JSON-encoded array. + assert.Equal(t, `["abc"]`, wikiJSONContainsArg("postgres", `["abc"]`, "abc")) + assert.Equal(t, `["abc"]`, wikiJSONContainsArg("mysql", `["abc"]`, "abc")) + // SQLite takes the bare scalar. + assert.Equal(t, "abc", wikiJSONContainsArg("sqlite", `["abc"]`, "abc")) +} + +func TestWikiJSONAsText(t *testing.T) { + assert.Equal(t, "source_refs::text", wikiJSONAsText("postgres", "source_refs")) + assert.Equal(t, "CAST(source_refs AS CHAR)", wikiJSONAsText("mysql", "source_refs")) + assert.Equal(t, "CAST(source_refs AS TEXT)", wikiJSONAsText("sqlite", "source_refs")) +} + +func TestWikiJSONEqual(t *testing.T) { + assert.Equal(t, "category_path::jsonb = ?::jsonb", wikiJSONEqual("postgres", "category_path")) + assert.Equal(t, "category_path = CAST(? AS JSON)", wikiJSONEqual("mysql", "category_path")) + assert.Equal(t, "category_path = ?", wikiJSONEqual("sqlite", "category_path")) +} + +func TestWikiCaseInsensitiveRegex(t *testing.T) { + assert.Equal(t, "title ~* ?", wikiCaseInsensitiveRegex("postgres", "title", "?")) + // MySQL uses REGEXP_LIKE with the 'i' match-type flag so the + // case-insensitivity is explicit and the bound pattern is treated as a + // real regex (the bare `col REGEXP ?` form is case-sensitive under the + // default utf8mb4_general_ci collation and has inconsistent behaviour + // across MySQL point releases). + assert.Equal(t, "REGEXP_LIKE(title, ?, 'i')", wikiCaseInsensitiveRegex("mysql", "title", "?")) + // SQLite falls back to LIKE (no built-in REGEXP). This is a substring + // approximation, NOT a superset of regex - it has different matching + // semantics (e.g. no alternation, no anchors). + got := wikiCaseInsensitiveRegex("sqlite", "title", "?") + assert.True(t, strings.HasPrefix(got, "title LIKE"), "sqlite should fall back to LIKE; got %s", got) +} + +func TestWikiSimilarityRank(t *testing.T) { + pg := wikiSimilarityRank("postgres", "title", "?") + assert.True(t, strings.HasPrefix(pg, "similarity(lower(title)")) + my := wikiSimilarityRank("mysql", "title", "?") + assert.Contains(t, my, "LOWER(title) LIKE CONCAT") + // SQLite / unknown -> LIKE-based. + got := wikiSimilarityRank("sqlite", "title", "?") + assert.Contains(t, got, "LOWER(title) LIKE") +} + +func TestWikiSimilarityThreshold(t *testing.T) { + assert.Contains(t, wikiSimilarityThreshold("postgres", "title", "?"), "lower(title) % ?") + assert.Contains(t, wikiSimilarityThreshold("mysql", "title", "?"), "LOWER(title) LIKE CONCAT") + assert.Contains(t, wikiSimilarityThreshold("sqlite", "title", "?"), "LOWER(title) LIKE") +} + +func TestWikiFullTextSearch(t *testing.T) { + frag := wikiFullTextSearch("postgres") + assert.Contains(t, frag, "to_tsvector") + assert.Contains(t, frag, "plainto_tsquery") + + frag = wikiFullTextSearch("mysql") + assert.Contains(t, frag, "LOWER(title)") + assert.Contains(t, frag, "LOWER(content)") + assert.Contains(t, frag, "LOWER(aliases)") + + // SQLite: same shape as MySQL (multi-column LIKE). + frag = wikiFullTextSearch("sqlite") + assert.Contains(t, frag, "LOWER(title)") +} diff --git a/internal/application/repository/wiki_page.go b/internal/application/repository/wiki_page.go index 4cbdc97c32..c941e63e4e 100644 --- a/internal/application/repository/wiki_page.go +++ b/internal/application/repository/wiki_page.go @@ -31,17 +31,12 @@ func NewWikiPageRepository(db *gorm.DB) interfaces.WikiPageRepository { } func (r *wikiPageRepository) wikiCategoryRankOrder() string { - if r.db != nil && r.db.Dialector != nil && r.db.Dialector.Name() == "sqlite" { - return "CASE WHEN COALESCE(json_array_length(category_path), 0) > 0 THEN 0 ELSE 1 END ASC" - } - return "CASE WHEN COALESCE(jsonb_array_length(category_path), 0) > 0 THEN 0 ELSE 1 END ASC" + dialect := wikiDialectName(r.db) + return "CASE WHEN " + wikiJSONArrayLength(dialect, "category_path") + " > 0 THEN 0 ELSE 1 END ASC" } func (r *wikiPageRepository) wikiEmptyInLinksPredicate() string { - if r.db != nil && r.db.Dialector != nil && r.db.Dialector.Name() == "sqlite" { - return "(in_links IS NULL OR json_array_length(in_links) = 0)" - } - return "(in_links IS NULL OR in_links = '[]'::JSONB)" + return wikiJSONArrayLength(wikiDialectName(r.db), "in_links") + " = 0" } // Create inserts a new wiki page record @@ -319,12 +314,25 @@ func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRe query = query.Where("status = ?", req.Status) } if req.Query != "" { - // Use PostgreSQL full-text search + ILIKE for aliases - query = query.Where( - "(to_tsvector('simple', coalesce(title, '') || ' ' || coalesce(content, '')) @@ plainto_tsquery('simple', ?) OR aliases::text ILIKE ?)", - req.Query, - "%"+req.Query+"%", - ) + // Dialect-aware full-text search. PostgreSQL keeps to_tsvector / + // plainto_tsquery (PG-specific GIN-backed full-text). MySQL / + // SQLite use multi-column LOWER() LIKE LOWER() substring matching. + // Its matching and ranking semantics are not equivalent to PostgreSQL + // full-text search, but the query remains executable on every supported + // metadata database. + dialect := wikiDialectName(r.db) + frag := wikiFullTextSearch(dialect) + // PG's to_tsvector takes the raw query (plainto_tsquery does its + // own tokenisation); its aliases ILIKE branch and every MySQL / + // SQLite LIKE column want %query%. + var args []interface{} + if dialect == "postgres" { + args = []interface{}{req.Query, "%" + req.Query + "%"} + } else { + pattern := "%" + req.Query + "%" + args = []interface{}{pattern, pattern, pattern} + } + query = query.Where(frag, args...) } // Directory filters are pushed to SQL so the DB does the counting and // pagination instead of loading every page of the type into memory. `depth` @@ -340,11 +348,8 @@ func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRe } if wantPath := types.CleanWikiCategoryPath(req.CategoryPath); len(wantPath) > 0 { if encoded, err := json.Marshal([]string(wantPath)); err == nil { - if r.db.Dialector != nil && r.db.Dialector.Name() == "postgres" { - query = query.Where("category_path::jsonb = ?::jsonb", string(encoded)) - } else { - query = query.Where("category_path = ?", string(encoded)) - } + dialect := wikiDialectName(r.db) + query = query.Where(wikiJSONEqual(dialect, "category_path"), string(encoded)) } } @@ -462,6 +467,11 @@ func (r *wikiPageRepository) ListByTypeLight( // ListBySourceRef retrieves all wiki pages that reference a given source knowledge ID. // Handles both old format ("knowledgeID") and new format ("knowledgeID|title") in source_refs JSON array. +// +// Dialect-aware: PostgreSQL uses jsonb containment (@> ?::jsonb), MySQL +// uses JSON_CONTAINS, SQLite uses json_each. The legacy "knowledgeID|title" +// prefix form falls back to a text LIKE on the serialized JSON, which is +// portable across all three dialects. func (r *wikiPageRepository) ListBySourceRef(ctx context.Context, kbID string, sourceKnowledgeID string) ([]*types.WikiPage, error) { // Build the JSON needle safely so arbitrary IDs cannot break out of the // quoted string (e.g. ids containing quotes or backslashes). @@ -486,11 +496,16 @@ func (r *wikiPageRepository) ListBySourceRef(ctx context.Context, kbID string, s // with %…% to match anywhere in the serialized JSON array. likePattern := "%" + escapeLikePattern(prefixStr) + "%" + dialect := wikiDialectName(r.db) + containsFrag := wikiJSONContains(dialect, "source_refs") + containsArg := wikiJSONContainsArg(dialect, string(needle), sourceKnowledgeID) + textFrag := wikiJSONAsText(dialect, "source_refs") + " LIKE ?" + var pages []*types.WikiPage if err := r.db.WithContext(ctx). - Where("knowledge_base_id = ? AND (source_refs @> ?::jsonb OR source_refs::text LIKE ?)", + Where("knowledge_base_id = ? AND ("+containsFrag+" OR "+textFrag+")", kbID, - string(needle), + containsArg, likePattern, ). Find(&pages).Error; err != nil { @@ -523,12 +538,17 @@ func (r *wikiPageRepository) ListSlugsBySourceRef(ctx context.Context, kbID stri } likePattern := "%" + escapeLikePattern(prefixStr) + "%" + dialect := wikiDialectName(r.db) + containsFrag := wikiJSONContains(dialect, "source_refs") + containsArg := wikiJSONContainsArg(dialect, string(needle), sourceKnowledgeID) + textFrag := wikiJSONAsText(dialect, "source_refs") + " LIKE ?" + var slugs []string if err := r.db.WithContext(ctx). Model(&types.WikiPage{}). - Where("knowledge_base_id = ? AND (source_refs @> ?::jsonb OR source_refs::text LIKE ?)", + Where("knowledge_base_id = ? AND ("+containsFrag+" OR "+textFrag+")", kbID, - string(needle), + containsArg, likePattern, ). Pluck("slug", &slugs).Error; err != nil { @@ -696,7 +716,29 @@ func (r *wikiPageRepository) DeleteFolder(ctx context.Context, kbID string, id s // Keep the emptiness test in the same SQL statement as the soft delete. // A page move or child-folder create can race the service's earlier checks; // a check-then-delete sequence would otherwise leave a dangling folder_id. - result := r.db.WithContext(ctx).Exec(` + var result *gorm.DB + if wikiDialectName(r.db) == "mysql" { + // MySQL rejects an UPDATE whose WHERE subquery reads the target table + // (error 1093). A self-join UPDATE expresses the same atomic guard + // without reopening wiki_folders from a subquery. + result = r.db.WithContext(ctx).Exec(` +UPDATE wiki_folders AS target +LEFT JOIN wiki_pages AS page + ON page.knowledge_base_id = target.knowledge_base_id + AND page.folder_id = target.id + AND page.deleted_at IS NULL +LEFT JOIN wiki_folders AS child + ON child.knowledge_base_id = target.knowledge_base_id + AND child.parent_id = target.id + AND child.deleted_at IS NULL +SET target.deleted_at = ? +WHERE target.knowledge_base_id = ? + AND target.id = ? + AND target.deleted_at IS NULL + AND page.id IS NULL + AND child.id IS NULL`, time.Now().UTC(), kbID, id) + } else { + result = r.db.WithContext(ctx).Exec(` UPDATE wiki_folders SET deleted_at = ? WHERE knowledge_base_id = ? AND id = ? AND deleted_at IS NULL @@ -707,7 +749,8 @@ WHERE knowledge_base_id = ? AND id = ? AND deleted_at IS NULL AND NOT EXISTS ( SELECT 1 FROM wiki_folders AS child WHERE child.knowledge_base_id = ? AND child.parent_id = ? AND child.deleted_at IS NULL - )`, time.Now(), kbID, id, kbID, id, kbID, id) + )`, time.Now().UTC(), kbID, id, kbID, id, kbID, id) + } if result.Error != nil { return result.Error } @@ -817,6 +860,9 @@ func (r *wikiPageRepository) ListSummariesByKnowledgeIDs( // Build OR clauses without using overly-clever GORM tricks: assemble // raw SQL fragments + args. Keeping this defensive because source_refs // patterns include user-controlled knowledge ids. + dialect := wikiDialectName(r.db) + containsFrag := wikiJSONContains(dialect, "source_refs") + textFrag := wikiJSONAsText(dialect, "source_refs") + " LIKE ?" clauses := make([]string, 0, len(kids)*2) args := make([]interface{}, 0, len(kids)*2) for _, kid := range kids { @@ -827,8 +873,8 @@ func (r *wikiPageRepository) ListSummariesByKnowledgeIDs( if err != nil { return nil, fmt.Errorf("marshal kid needle: %w", err) } - clauses = append(clauses, "source_refs @> ?::jsonb") - args = append(args, string(needle)) + clauses = append(clauses, containsFrag) + args = append(args, wikiJSONContainsArg(dialect, string(needle), kid)) prefix, err := json.Marshal(kid + "|") if err != nil { @@ -838,7 +884,7 @@ func (r *wikiPageRepository) ListSummariesByKnowledgeIDs( if len(prefixStr) >= 2 && prefixStr[len(prefixStr)-1] == '"' { prefixStr = prefixStr[:len(prefixStr)-1] } - clauses = append(clauses, "source_refs::text LIKE ?") + clauses = append(clauses, textFrag) args = append(args, "%"+escapeLikePattern(prefixStr)+"%") } if len(clauses) == 0 { @@ -1004,16 +1050,14 @@ func (r *wikiPageRepository) ListByTypeRecent( } // FindSimilarPages returns the top-k entity/concept pages whose lowercase -// title is most similar to the given query under PostgreSQL pg_trgm -// trigram similarity. Backed by idx_wiki_pages_title_trgm (GIN -// gin_trgm_ops, migration 000041). Used by the dedup pre-filter to -// surface candidate merge targets without loading every entity/concept -// page into Go. +// title matches the query. PostgreSQL uses pg_trgm similarity (backed by +// idx_wiki_pages_title_trgm); MySQL and SQLite use a deterministic +// contains-match fallback. Used by the dedup pre-filter to surface candidate +// merge targets without loading every entity/concept page into Go. // // types is an optional page_type allow-list; empty means entity+concept. -// limit is clamped to [1, 50]. Pages whose title similarity is below -// 0.1 are dropped server-side via the `%` operator (which respects -// pg_trgm.similarity_threshold). +// limit is clamped to [1, 50]. PostgreSQL applies its configured +// pg_trgm.similarity_threshold; the fallback dialects require containment. func (r *wikiPageRepository) FindSimilarPages( ctx context.Context, kbID string, @@ -1036,13 +1080,31 @@ func (r *wikiPageRepository) FindSimilarPages( q := strings.ToLower(strings.TrimSpace(query)) + dialect := wikiDialectName(r.db) + simRank := wikiSimilarityRank(dialect, "title", "?") + simThreshold := wikiSimilarityThreshold(dialect, "title", "?") + // Order clause: under PG, similarity() yields a graded rank so + // `sim DESC` produces a meaningful ordering. Under MySQL/SQLite the + // helper degrades to a binary 0/1 contains-match, so every matching + // row ties at sim=1 and the ORDER BY needs deterministic tiebreakers + // (title length ascending = prefer concise titles, then updated_at + // descending = prefer recently-edited pages) to avoid nondeterministic + // result order across calls. + orderClause := "sim DESC" + switch dialect { + case "mysql": + orderClause = "sim DESC, CHAR_LENGTH(title) ASC, updated_at DESC" + case "sqlite": + orderClause = "sim DESC, LENGTH(title) ASC, updated_at DESC" + } + var rows []types.WikiPageLite if err := r.db.WithContext(ctx). Model(&types.WikiPage{}). - Select("slug, title, page_type, status, aliases, out_links, similarity(lower(title), ?) AS sim", q). - Where("knowledge_base_id = ? AND page_type IN ? AND status <> ? AND lower(title) % ?", + Select("slug, title, page_type, status, aliases, out_links, "+simRank+" AS sim", q). + Where("knowledge_base_id = ? AND page_type IN ? AND status <> ? AND "+simThreshold, kbID, pageTypes, types.WikiPageStatusArchived, q). - Order("sim DESC"). + Order(orderClause). Limit(limit). Scan(&rows).Error; err != nil { return nil, err @@ -1135,8 +1197,9 @@ func escapeLikePattern(s string) string { return replacer.Replace(s) } -// Search performs case-insensitive POSIX regex search on wiki pages within a knowledge base. -// The query is interpreted as a PostgreSQL regular expression (via ~*). +// Search performs a case-insensitive pattern search on wiki pages within a +// knowledge base. PostgreSQL uses ~*, MySQL uses REGEXP_LIKE, and SQLite +// retains its historical LIKE fallback. // // Results are ranked by where the query hit, highest-relevance first: // @@ -1162,17 +1225,29 @@ func (r *wikiPageRepository) Search(ctx context.Context, kbID string, query stri // alias so the DB only computes the rank once. Parameterized four // times with the same regex to avoid coupling to GORM's positional // arg rewriting quirks. + // + // Dialect-aware: PostgreSQL keeps the ~* regex operator (pg_trgm GIN + // index can serve it). MySQL uses REGEXP_LIKE with an explicit + // case-insensitive flag. SQLite falls back to LIKE with different matching + // semantics. + dialect := wikiDialectName(r.db) rankExpr := "CASE " + - "WHEN title ~* ? THEN 4 " + - "WHEN slug ~* ? THEN 3 " + - "WHEN summary ~* ? THEN 2 " + - "WHEN content ~* ? THEN 1 " + + "WHEN " + wikiCaseInsensitiveRegex(dialect, "title", "?") + " THEN 4 " + + "WHEN " + wikiCaseInsensitiveRegex(dialect, "slug", "?") + " THEN 3 " + + "WHEN " + wikiCaseInsensitiveRegex(dialect, "summary", "?") + " THEN 2 " + + "WHEN " + wikiCaseInsensitiveRegex(dialect, "content", "?") + " THEN 1 " + "ELSE 0 END AS match_rank" + matchPred := "(" + + wikiCaseInsensitiveRegex(dialect, "title", "?") + " OR " + + wikiCaseInsensitiveRegex(dialect, "content", "?") + " OR " + + wikiCaseInsensitiveRegex(dialect, "summary", "?") + " OR " + + wikiCaseInsensitiveRegex(dialect, "slug", "?") + ")" + var pages []*types.WikiPage if err := r.db.WithContext(ctx). Select("*, "+rankExpr, query, query, query, query). - Where("knowledge_base_id = ? AND (title ~* ? OR content ~* ? OR summary ~* ? OR slug ~* ?)", + Where("knowledge_base_id = ? AND "+matchPred, kbID, query, query, query, query). Where("status != ?", "archived"). Order("match_rank DESC, updated_at DESC"). diff --git a/internal/application/repository/wiki_page_test.go b/internal/application/repository/wiki_page_test.go index ef8f0952ce..12f6a4958c 100644 --- a/internal/application/repository/wiki_page_test.go +++ b/internal/application/repository/wiki_page_test.go @@ -379,6 +379,32 @@ func TestListPagesCursorExcludesArchivedPages(t *testing.T) { assert.Empty(t, next) } +func TestFindSimilarPagesSQLiteReturnsRankedMatches(t *testing.T) { + db := setupWikiPagesTestDB(t) + repo := NewWikiPageRepository(db) + ctx := context.Background() + + for _, page := range []*types.WikiPage{ + makeWikiPage("kb-similar", "entity/alpha", types.WikiPageTypeEntity, types.WikiPageStatusPublished), + makeWikiPage("kb-similar", "entity/alpha-longer-title", types.WikiPageTypeEntity, types.WikiPageStatusPublished), + makeWikiPage("kb-similar", "entity/unrelated", types.WikiPageTypeEntity, types.WikiPageStatusPublished), + } { + require.NoError(t, repo.Create(ctx, page)) + } + + got, err := repo.FindSimilarPages( + ctx, + "kb-similar", + "alpha", + []string{types.WikiPageTypeEntity}, + 10, + ) + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, "alpha", got[0].Title) + assert.Equal(t, "alpha-longer-title", got[1].Title) +} + // TestListByTypeLight_ClampsLimit verifies the [1, 200] clamp. We don't // want a client passing limit=100000 and forcing the DB to return a // multi-MB response. diff --git a/internal/application/service/agent_service.go b/internal/application/service/agent_service.go index 171420eaae..e146385c92 100644 --- a/internal/application/service/agent_service.go +++ b/internal/application/service/agent_service.go @@ -43,6 +43,19 @@ func dedupStrings(in []string) []string { return out } +func filterDatabaseCompatibleTools(allowedTools []string, dialect string) ([]string, []string) { + filtered := make([]string, 0, len(allowedTools)) + dropped := make([]string, 0, 1) + for _, toolName := range allowedTools { + if toolName == tools.ToolDatabaseQuery && !tools.DatabaseQuerySupported(dialect) { + dropped = append(dropped, toolName) + continue + } + filtered = append(filtered, toolName) + } + return filtered, dropped +} + // agentHasKnowledgeScope reports whether the agent has any KB retrieval scope for // this turn. Tag-only @mentions populate SearchTargets (with TagIDs) but leave // KnowledgeBases / KnowledgeIDs empty — those must still count as in-scope. @@ -580,6 +593,21 @@ func (s *agentService) registerTools( } } + dialect := "" + if s.db != nil && s.db.Dialector != nil { + dialect = s.db.Dialector.Name() + } + var droppedForDialect []string + allowedTools, droppedForDialect = filterDatabaseCompatibleTools(allowedTools, dialect) + if len(droppedForDialect) > 0 { + logger.Warnf( + ctx, + "Dropped tools %v because metadata database dialect %q is unsupported", + droppedForDialect, + dialect, + ) + } + // Deduplicate while preserving original order. allowedTools = dedupStrings(allowedTools) diff --git a/internal/application/service/agent_service_dialect_test.go b/internal/application/service/agent_service_dialect_test.go new file mode 100644 index 0000000000..5b551ae3db --- /dev/null +++ b/internal/application/service/agent_service_dialect_test.go @@ -0,0 +1,35 @@ +package service + +import ( + "testing" + + "github.com/Tencent/WeKnora/internal/agent/tools" + "github.com/stretchr/testify/require" +) + +func TestFilterDatabaseCompatibleToolsDropsDatabaseQueryForMySQL(t *testing.T) { + allowed := []string{tools.ToolThinking, tools.ToolDatabaseQuery, tools.ToolKnowledgeSearch} + + got, dropped := filterDatabaseCompatibleTools(allowed, "mysql") + + require.Equal(t, []string{tools.ToolThinking, tools.ToolKnowledgeSearch}, got) + require.Equal(t, []string{tools.ToolDatabaseQuery}, dropped) +} + +func TestFilterDatabaseCompatibleToolsKeepsDatabaseQueryForPostgres(t *testing.T) { + allowed := []string{tools.ToolThinking, tools.ToolDatabaseQuery} + + got, dropped := filterDatabaseCompatibleTools(allowed, "postgres") + + require.Equal(t, allowed, got) + require.Empty(t, dropped) +} + +func TestFilterDatabaseCompatibleToolsKeepsDatabaseQueryForSQLite(t *testing.T) { + allowed := []string{tools.ToolThinking, tools.ToolDatabaseQuery} + + got, dropped := filterDatabaseCompatibleTools(allowed, "sqlite") + + require.Equal(t, allowed, got) + require.Empty(t, dropped) +} diff --git a/internal/application/service/message.go b/internal/application/service/message.go index bc6adfad4d..3ba5af29a7 100644 --- a/internal/application/service/message.go +++ b/internal/application/service/message.go @@ -502,7 +502,7 @@ func (s *messageService) SearchMessages(ctx context.Context, params *types.Messa var vectorResults []*types.MessageSearchResultItem var err error - // Step 1: Keyword search (direct PG ILIKE) + // Step 1: Keyword search (dialect-aware case-insensitive LIKE) if params.Mode == types.MessageSearchModeKeyword || params.Mode == types.MessageSearchModeHybrid { keywordResults, err = s.messageRepo.SearchMessagesByKeyword(ctx, tenantID, params.Query, params.SessionIDs, params.Limit*3) if err != nil { diff --git a/internal/application/service/storagebackend.go b/internal/application/service/storagebackend.go index 0676cf52e1..d63d26ebad 100644 --- a/internal/application/service/storagebackend.go +++ b/internal/application/service/storagebackend.go @@ -132,7 +132,7 @@ func (s *StorageBackendService) Delete(ctx context.Context, tenantID uint64, id return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { var backend types.StorageBackend query := tx.Where("tenant_id = ? AND id = ?", tenantID, id) - if tx.Dialector.Name() == "postgres" { + if dialectSupportsRowLocking(tx.Dialector.Name()) { query = query.Clauses(clause.Locking{Strength: "UPDATE"}) } if err := query.First(&backend).Error; err != nil { @@ -178,7 +178,7 @@ func (s *StorageBackendService) SetDefault(ctx context.Context, tenantID uint64, return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { var backend types.StorageBackend query := tx.Where("tenant_id = ? AND id = ?", tenantID, id) - if tx.Dialector.Name() == "postgres" { + if dialectSupportsRowLocking(tx.Dialector.Name()) { query = query.Clauses(clause.Locking{Strength: "UPDATE"}) } if err := query.First(&backend).Error; err != nil { diff --git a/internal/application/service/vectorstore.go b/internal/application/service/vectorstore.go index 6ec25587cd..cdf2cd8932 100644 --- a/internal/application/service/vectorstore.go +++ b/internal/application/service/vectorstore.go @@ -189,10 +189,11 @@ func (s *vectorStoreService) DeleteStore(ctx context.Context, tenantID uint64, i err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { // tx inherits ctx from WithContext above; no need to re-attach. - // 1. Lock the store row (PG row-level X-lock; skipped on SQLite). + // 1. Lock the store row (row-level X-lock on postgres/mysql; + // skipped on SQLite which has no FOR UPDATE). var store types.VectorStore q := tx.Where("id = ? AND tenant_id = ?", id, tenantID) - if s.isPostgres(tx) { + if s.supportsRowLocking(tx) { q = q.Clauses(clause.Locking{Strength: "UPDATE"}) } if err := q.First(&store).Error; err != nil { @@ -247,11 +248,22 @@ func (s *vectorStoreService) unregisterSafely(ctx context.Context, storeID strin } } -// isPostgres reports whether the active GORM dialector is PostgreSQL. -// Used to gate dialect-specific clauses (e.g., SELECT FOR UPDATE) that -// SQLite would either ignore (recent versions) or fail to compile on. -func (s *vectorStoreService) isPostgres(db *gorm.DB) bool { - return db != nil && db.Dialector != nil && db.Dialector.Name() == "postgres" +// dialectSupportsRowLocking reports whether a GORM dialector with the +// given name supports SELECT ... FOR UPDATE row-level locking. +// PostgreSQL and MySQL both do; SQLite does not (it serialises via +// database-level locking instead). Unknown dialects are treated as +// not supporting it — they must opt in explicitly. +func dialectSupportsRowLocking(dialectName string) bool { + return dialectName == "postgres" || dialectName == "mysql" +} + +// supportsRowLocking reports whether the active GORM dialector supports +// SELECT ... FOR UPDATE. Used to gate the row-level X-lock that +// serialises vector-store deletes against KB binding count reads. +// Renamed from isPostgres because the contract is about a capability +// (row locking), not a dialect identity — MySQL also takes the lock. +func (s *vectorStoreService) supportsRowLocking(db *gorm.DB) bool { + return db != nil && db.Dialector != nil && dialectSupportsRowLocking(db.Dialector.Name()) } // SaveDetectedVersion updates the connection_config.version for a stored vector store. diff --git a/internal/application/service/vectorstore_rowlock_test.go b/internal/application/service/vectorstore_rowlock_test.go new file mode 100644 index 0000000000..e72a4de961 --- /dev/null +++ b/internal/application/service/vectorstore_rowlock_test.go @@ -0,0 +1,45 @@ +package service + +import ( + "testing" +) + +// dialectSupportsRowLocking reports whether a GORM dialector with the +// given name supports SELECT ... FOR UPDATE row-level locking. It +// backs vectorStoreService.supportsRowLocking (renamed from +// isPostgres), which gates the clause.Locking{Strength: "UPDATE"} +// clause in DeleteStore. +// +// PostgreSQL and MySQL both support FOR UPDATE. SQLite does not (it +// serialises via database-level locking), so the clause is skipped to +// avoid SQL parse errors on older SQLite versions. +// +// The rename from isPostgres is deliberate: the method's contract is +// about a capability (row locking), not a dialect identity. Under +// DB_DRIVER=mysql the delete guard must still take the row lock — +// otherwise concurrent deletes can pass the KB-count check +// simultaneously and leave dangling bindings. +// +// These tests are written before the helper exists (TDD red). + +func TestDialectSupportsRowLocking(t *testing.T) { + tests := []struct { + dialect string + want bool + }{ + {"postgres", true}, + {"mysql", true}, + {"sqlite", false}, + {"", false}, + {"sqlserver", false}, + } + + for _, tt := range tests { + t.Run(tt.dialect, func(t *testing.T) { + got := dialectSupportsRowLocking(tt.dialect) + if got != tt.want { + t.Fatalf("dialectSupportsRowLocking(%q) = %v; want %v", tt.dialect, got, tt.want) + } + }) + } +} diff --git a/internal/application/service/wiki_page.go b/internal/application/service/wiki_page.go index f523c312f6..cfae25fbd9 100644 --- a/internal/application/service/wiki_page.go +++ b/internal/application/service/wiki_page.go @@ -974,8 +974,8 @@ func (s *wikiPageService) ListByTypeRecent(ctx context.Context, kbID string, pag return s.repo.ListByTypeRecent(ctx, kbID, pageType, limit) } -// FindSimilarPages performs a pg_trgm similarity search; used by the -// dedup pre-filter to surface candidate merge targets. +// FindSimilarPages delegates to the repository's dialect-aware candidate +// matching; used by the dedup pre-filter to surface merge targets. func (s *wikiPageService) FindSimilarPages(ctx context.Context, kbID string, query string, pageTypes []string, limit int) ([]*types.WikiPageLite, error) { return s.repo.FindSimilarPages(ctx, kbID, query, pageTypes, limit) } diff --git a/internal/container/container.go b/internal/container/container.go index 8a9265f5ee..daf948eea4 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -29,6 +29,7 @@ import ( "github.com/redis/go-redis/v9" "go.uber.org/dig" "google.golang.org/grpc" + "gorm.io/driver/mysql" "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" @@ -53,6 +54,7 @@ import ( "github.com/Tencent/WeKnora/internal/common" "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/database" + "github.com/Tencent/WeKnora/internal/database/mysqlconfig" "github.com/Tencent/WeKnora/internal/datasource" feishuConnector "github.com/Tencent/WeKnora/internal/datasource/connector/feishu" notionConnector "github.com/Tencent/WeKnora/internal/datasource/connector/notion" @@ -561,7 +563,7 @@ func initRedisClient() (*redis.Client, error) { // initDatabase initializes database connection // Creates and configures database connection based on environment configuration -// Supports multiple database backends (PostgreSQL) +// Supports PostgreSQL, SQLite, and MySQL metadata databases. // Parameters: // - cfg: Application configuration // @@ -569,8 +571,14 @@ func initRedisClient() (*redis.Client, error) { // - Configured database connection // - Error if connection fails func initDatabase(cfg *config.Config) (*gorm.DB, error) { + if err := ValidateDriverCombination(os.Getenv("DB_DRIVER"), os.Getenv("RETRIEVE_DRIVER")); err != nil { + return nil, err + } + var dialector gorm.Dialector var migrateDSN string + var mysqlPoolCfg mysqlconfig.PoolConfig + var mysqlMigrationDSN string var sqliteDBPath string switch os.Getenv("DB_DRIVER") { case "postgres": @@ -632,6 +640,18 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { sqliteDBPath = dbPath migrateDSN = "sqlite3://" + dbPath logger.Infof(context.Background(), "DB Config: driver=sqlite path=%s", dbPath) + case "mysql": + // MySQL takes over the metadata layer only; vector retrieval + // must be delegated to an external engine via RETRIEVE_DRIVER. + applicationDSN, migrationDSN, mysqlPool, dsnErr := mysqlconfig.BuildDSN(os.Getenv) + if dsnErr != nil { + return nil, fmt.Errorf("invalid MySQL DSN configuration: %w", dsnErr) + } + dialector = mysql.Open(applicationDSN) + mysqlMigrationDSN = migrationDSN + mysqlPoolCfg = mysqlPool + logger.Infof(context.Background(), "DB Config: driver=mysql host=%s port=%s dbname=%s", + os.Getenv("DB_HOST"), os.Getenv("DB_PORT"), os.Getenv("DB_NAME")) default: return nil, fmt.Errorf("unsupported database driver: %s", os.Getenv("DB_DRIVER")) } @@ -646,14 +666,14 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { // Sanity check: dialect-specific code in services (notably the // vector_stores delete guard) compares Dialector.Name() to "postgres" / - // "sqlite" string literals. A future driver swap that produces a - // different name (e.g., a wrapper dialect for managed PG) would silently - // fall back to the SQLite path, dropping the row-level X-lock. Catching - // the mismatch at startup is loud and inexpensive. - if name := db.Dialector.Name(); name != "postgres" && name != "sqlite" { + // "sqlite" / "mysql" string literals. A future driver swap that produces + // a different name (e.g., a wrapper dialect for managed PG) would + // silently fall back to the SQLite path, dropping the row-level X-lock. + // Catching the mismatch at startup is loud and inexpensive. + if name := db.Dialector.Name(); name != "postgres" && name != "sqlite" && name != "mysql" { return nil, fmt.Errorf( - "unsupported gorm dialector %q; expected postgres or sqlite "+ - "(see vectorStoreService.isPostgres for impact)", name) + "unsupported gorm dialector %q; expected postgres, sqlite, or mysql "+ + "(see vectorStoreService.supportsRowLocking for impact)", name) } if os.Getenv("DB_DRIVER") == "sqlite" { @@ -675,13 +695,34 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { autoRecover := os.Getenv("AUTO_RECOVER_DIRTY") != "false" migrationOpts := database.MigrationOptions{ AutoRecoverDirty: autoRecover, - SQLiteDBPath: sqliteDBPath, + // MySQL DDL is not transactional, so auto-recovering a dirty + // migration by forcing the version backward and re-running Up() + // can leave a half-applied schema. Fail-closed for MySQL and let + // the operator inspect the schema manually. + FailOnDirty: os.Getenv("DB_DRIVER") == "mysql", + SQLiteDBPath: sqliteDBPath, + MySQLMigrationDSN: mysqlMigrationDSN, } // Run base migrations (all versioned migrations including embeddings) // The embeddings migration will be conditionally executed based on skip_embedding parameter in DSN if err := database.RunMigrationsWithOptions(migrateDSN, migrationOpts); err != nil { - // Log warning but don't fail startup - migrations might be handled externally + // Migration failure handling is dialect-aware: + // + // - MySQL: a fresh deployment with a half-applied schema is a + // brick - every business query will fail. Fail startup so the + // operator notices immediately. Operators who run migrations + // out-of-band (e.g. a CI job with golang-migrate) set + // AUTO_MIGRATE=false (handled above), which skips the + // in-process attempt entirely - the canonical opt-out. + // - PostgreSQL / SQLite: preserve the historical "warn and + // continue" behaviour so an existing deployment that runs + // migrations via a separate job is not blocked by a startup + // gate. + if os.Getenv("DB_DRIVER") == "mysql" { + return nil, fmt.Errorf("database migration failed (refusing to start MySQL with a partial schema): %w "+ + "(set AUTO_MIGRATE=false if migrations are managed externally)", err) + } logger.Warnf(context.Background(), "Database migration failed: %v", err) logger.Warnf( context.Background(), @@ -712,13 +753,16 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { // Configure connection pool parameters if os.Getenv("DB_DRIVER") == "sqlite" { // SQLite only supports one concurrent writer even in WAL mode. - // Limiting to a single open connection serialises all DB access and - // prevents "database is locked" errors from concurrent goroutines. sqlDB.SetMaxOpenConns(1) + } else if os.Getenv("DB_DRIVER") == "mysql" { + sqlDB.SetMaxOpenConns(mysqlPoolCfg.MaxOpenConns) + sqlDB.SetMaxIdleConns(mysqlPoolCfg.MaxIdleConns) + sqlDB.SetConnMaxLifetime(mysqlPoolCfg.ConnMaxLifetime) + sqlDB.SetConnMaxIdleTime(mysqlPoolCfg.ConnMaxIdleTime) } else { sqlDB.SetMaxIdleConns(10) + sqlDB.SetConnMaxLifetime(time.Duration(10) * time.Minute) } - sqlDB.SetConnMaxLifetime(time.Duration(10) * time.Minute) return db, nil } @@ -726,6 +770,12 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { // resolveStorageProviderPending replaces the "__pending_env__" sentinel in // knowledge_bases.storage_provider_config with the actual STORAGE_TYPE from the environment. // This runs once after SQL migrations to bind historical KBs to their real storage provider. +// +// The JSON extraction in the WHERE clause is dialect-aware via +// database.JSONPathExpr: postgres uses ->>'provider', MySQL uses +// ->>'$.provider', SQLite uses json_extract(..., '$.provider'). The +// bare-key postgres form errors on MySQL with "Invalid JSON path +// expression" once any row has non-null JSON. func resolveStorageProviderPending(db *gorm.DB) { storageType := strings.TrimSpace(os.Getenv("STORAGE_TYPE")) if storageType == "" { @@ -733,14 +783,29 @@ func resolveStorageProviderPending(db *gorm.DB) { } storageType = strings.ToLower(storageType) + providerExpr, err := database.JSONPathExpr(db.Dialector.Name(), "storage_provider_config", "provider") + if err != nil { + logger.Warnf(context.Background(), "Failed to build storage provider JSON path expression: %v", err) + return + } + updateQuery := fmt.Sprintf( + "UPDATE knowledge_bases SET storage_provider_config = ? "+ + "WHERE storage_provider_config IS NOT NULL AND %s = '__pending_env__'", + providerExpr, + ) result := db.Exec( - `UPDATE knowledge_bases SET storage_provider_config = ? WHERE storage_provider_config IS NOT NULL AND storage_provider_config->>'provider' = '__pending_env__'`, + updateQuery, fmt.Sprintf(`{"provider":"%s"}`, storageType), ) if result.Error != nil { logger.Warnf(context.Background(), "Failed to resolve __pending_env__ storage providers: %v", result.Error) } else if result.RowsAffected > 0 { - logger.Infof(context.Background(), "Resolved %d knowledge bases with __pending_env__ storage provider → %s", result.RowsAffected, storageType) + logger.Infof( + context.Background(), + "Resolved %d knowledge bases with __pending_env__ storage provider -> %s", + result.RowsAffected, + storageType, + ) } // Sync PostgreSQL sequences with actual MAX values to prevent duplicate key @@ -1034,7 +1099,7 @@ func initRetrieveEngineRegistry( // is absent from this process, which happens when startup skipped it after // a construction failure or when another instance registered it. registry := retriever.NewRetrieveEngineRegistry(storeRepo, engineFactory) - retrieveDriver := strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",") + retrieveDriver := ParseRetrieveDrivers(os.Getenv("RETRIEVE_DRIVER")) log := logger.GetLogger(context.Background()) // Audit sink for OpenSearch driver events (index created / reindex). Driver // events fire under a tenant-scoped ctx at indexing time; the env-path diff --git a/internal/container/driver_validation.go b/internal/container/driver_validation.go new file mode 100644 index 0000000000..762409de31 --- /dev/null +++ b/internal/container/driver_validation.go @@ -0,0 +1,161 @@ +package container + +import ( + "fmt" + "slices" + "strings" + + "github.com/Tencent/WeKnora/internal/types" +) + +// localOnlyRetrieveEngines are the retriever engines that rely on a +// local embeddings table created by the DB_DRIVER's own migration set. +// They are only valid when DB_DRIVER is the matching local driver +// (postgres or sqlite). Under DB_DRIVER=mysql the embeddings table is +// never created, so any of these engines would crash at the first +// embedding write. +// +// The keys must match the keys of retrieverEngineMapping in +// internal/types/tenant.go. +var localOnlyRetrieveEngines = map[string]struct{}{ + "postgres": {}, + "sqlite": {}, +} + +// ParseRetrieveDrivers splits, trims, dedupes, and validates the RETRIEVE_DRIVER env value. +// Returns the normalized list of driver names. Used by both validation and registry registration. +func ParseRetrieveDrivers(retrieveDriver string) []string { + raw := strings.Split(retrieveDriver, ",") + seen := make(map[string]bool) + var result []string + for _, d := range raw { + trimmed := strings.TrimSpace(d) + if trimmed == "" || seen[trimmed] { + continue + } + seen[trimmed] = true + result = append(result, trimmed) + } + return result +} + +// ValidateDriverCombination checks that the DB_DRIVER and +// RETRIEVE_DRIVER environment variables are compatible. +// +// Rules: +// +// 1. DB_DRIVER=mysql cannot be paired with any local-only retriever +// (postgres or sqlite). The embeddings table is never created under +// MySQL mode (MySQL has no native vector type below 9.0, and the +// pgvector / ParadeDB stack is PostgreSQL-only), so a postgres or +// sqlite retriever would crash at the first embedding write. +// +// 2. DB_DRIVER=mysql requires at least one valid external retriever +// engine. An empty / whitespace-only RETRIEVE_DRIVER would let the +// app boot but every retrieval call would fail - fail fast instead. +// +// 3. Under DB_DRIVER=mysql every declared retriever must be a known +// key of the retriever registry (types.GetRetrieverEngineMapping()). +// This catches typos like "qdrnat" or stale names like "vikingdb" +// that the system would otherwise silently ignore. +// +// Non-mysql DB drivers are not validated here - the historical contract +// is that they fall through to per-engine validation downstream. We +// preserve that to avoid widening the blast radius of this guard. +// +// dbDriver is the value of DB_DRIVER ("postgres", "sqlite", "mysql"). +// retrieveDriver is the raw comma-separated RETRIEVE_DRIVER value +// (may be empty or contain surrounding whitespace). +// +// Returns nil for any combination that does not violate the rules. +func ValidateDriverCombination(dbDriver, retrieveDriver string) error { + if dbDriver != "mysql" { + return nil + } + + drivers := ParseRetrieveDrivers(retrieveDriver) + + // 1. Reject any local-only retriever (postgres / sqlite). + for _, d := range drivers { + if _, isLocal := localOnlyRetrieveEngines[d]; isLocal { + return fmt.Errorf( + "DB_DRIVER=mysql is incompatible with RETRIEVE_DRIVER=%s: "+ + "the %s retriever needs the embeddings table, which MySQL mode does not create. "+ + "Set RETRIEVE_DRIVER to an external engine instead "+ + "(%s).", + d, d, validExternalRetrieveEnginesHint(), + ) + } + } + + // 2. Reject empty / whitespace-only RETRIEVE_DRIVER. MySQL mode has + // no self-hosted embeddings, so an empty retriever is never usable. + if len(drivers) == 0 { + return fmt.Errorf( + "DB_DRIVER=mysql requires RETRIEVE_DRIVER to be set to at least one external engine "+ + "(%s); MySQL mode has no self-hosted embeddings table.", + validExternalRetrieveEnginesHint(), + ) + } + + // 3. Reject unknown engines. The names must match the keys of the + // retriever registry so the operator does not configure an engine + // the system will silently ignore. + registry := types.GetRetrieverEngineMapping() + for _, d := range drivers { + if _, known := registry[d]; !known { + return fmt.Errorf( + "DB_DRIVER=mysql: RETRIEVE_DRIVER entry %q is not a registered retriever engine. "+ + "Valid external engines: %s.", + d, validExternalRetrieveEnginesHint(), + ) + } + } + + // 4. Require at least one vector-capable engine. MySQL mode has no + // local embeddings table, so vector retrieval is mandatory. An + // engine like elasticsearch_v7 only supports keyword retrieval — + // accepting it would let the app boot but fail every vector query. + if !hasVectorCapableEngine(drivers, registry) { + return fmt.Errorf( + "DB_DRIVER=mysql requires at least one vector-capable retriever engine "+ + "(%s). All configured engines (%s) lack VectorRetrieverType capability.", + validExternalRetrieveEnginesHint(), strings.Join(drivers, ", "), + ) + } + + return nil +} + +// hasVectorCapableEngine returns true if at least one of the given drivers +// has VectorRetrieverType in its capability set. +func hasVectorCapableEngine(drivers []string, registry map[string][]types.RetrieverEngineParams) bool { + for _, d := range drivers { + if caps, ok := registry[d]; ok { + for _, c := range caps { + if c.RetrieverType == types.VectorRetrieverType { + return true + } + } + } + } + return false +} + +// validExternalRetrieveEnginesHint returns a human-readable list of +// retriever engines that are usable under DB_DRIVER=mysql. It is built +// from the actual registry (internal/types/tenant.go) minus the +// local-only engines, so the hint can never go stale or suggest a name +// the system does not recognise. +func validExternalRetrieveEnginesHint() string { + registry := types.GetRetrieverEngineMapping() + var names []string + for name := range registry { + if _, isLocal := localOnlyRetrieveEngines[name]; isLocal { + continue + } + names = append(names, name) + } + slices.Sort(names) + return strings.Join(names, ", ") +} diff --git a/internal/container/driver_validation_test.go b/internal/container/driver_validation_test.go new file mode 100644 index 0000000000..3b293216b8 --- /dev/null +++ b/internal/container/driver_validation_test.go @@ -0,0 +1,129 @@ +package container + +import ( + "slices" + "strings" + "testing" +) + +func TestValidateDriverCombination(t *testing.T) { + tests := []struct { + name string + dbDriver string + retrieveDriver string + wantErr bool + errContains string + }{ + { + name: "mysql rejects postgres local retriever", + dbDriver: "mysql", + retrieveDriver: "postgres", + wantErr: true, + errContains: "postgres", + }, + { + name: "mysql rejects sqlite local retriever", + dbDriver: "mysql", + retrieveDriver: "sqlite", + wantErr: true, + errContains: "sqlite", + }, + { + name: "mysql rejects empty retriever", + dbDriver: "mysql", + retrieveDriver: "", + wantErr: true, + errContains: "RETRIEVE_DRIVER", + }, + { + name: "mysql rejects unknown retriever", + dbDriver: "mysql", + retrieveDriver: "unknown-engine", + wantErr: true, + errContains: "not a registered retriever engine", + }, + { + name: "mysql rejects keyword-only retriever", + dbDriver: "mysql", + retrieveDriver: "elasticsearch_v7", + wantErr: true, + errContains: "vector-capable", + }, + { + name: "mysql accepts vector retriever", + dbDriver: "mysql", + retrieveDriver: "qdrant", + }, + { + name: "mysql accepts mixed keyword and vector retrievers", + dbDriver: "mysql", + retrieveDriver: "elasticsearch_v7,qdrant", + }, + { + name: "mysql normalizes whitespace empty entries and duplicates", + dbDriver: "mysql", + retrieveDriver: " qdrant, , qdrant ", + }, + { + name: "non-mysql driver passes through", + dbDriver: "postgres", + retrieveDriver: "unknown-engine", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateDriverCombination(tt.dbDriver, tt.retrieveDriver) + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), tt.errContains) { + t.Fatalf("error %q should contain %q", err.Error(), tt.errContains) + } + return + } + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + }) + } +} + +func TestParseRetrieveDrivers(t *testing.T) { + tests := []struct { + name string + in string + want []string + }{ + { + name: "splits and trims", + in: " qdrant , milvus ", + want: []string{"qdrant", "milvus"}, + }, + { + name: "drops empty entries", + in: ",qdrant, ,milvus,", + want: []string{"qdrant", "milvus"}, + }, + { + name: "deduplicates preserving order", + in: "qdrant, milvus, qdrant, milvus", + want: []string{"qdrant", "milvus"}, + }, + { + name: "all empty returns nil", + in: " , ", + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ParseRetrieveDrivers(tt.in) + if !slices.Equal(got, tt.want) { + t.Fatalf("ParseRetrieveDrivers(%q) = %v; want %v", tt.in, got, tt.want) + } + }) + } +} diff --git a/internal/container/resolve_storage_provider_test.go b/internal/container/resolve_storage_provider_test.go new file mode 100644 index 0000000000..39c0f0195c --- /dev/null +++ b/internal/container/resolve_storage_provider_test.go @@ -0,0 +1,53 @@ +package container + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// TestResolveStorageProviderPending_SQLite exercises the dialect-aware +// JSON path in resolveStorageProviderPending. SQLite stores JSON as TEXT +// and uses json_extract(..., '$.provider'); the historical code used +// PostgreSQL's ->>'provider' which errors on MySQL once any row has +// non-null JSON. +func TestResolveStorageProviderPending_SQLite(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.Exec(`CREATE TABLE knowledge_bases ( + id TEXT PRIMARY KEY, + storage_provider_config TEXT + )`).Error) + + // Seed three rows: one with the sentinel, one with a real provider, + // one NULL. Only the sentinel row should be rewritten. + require.NoError(t, db.Exec(`INSERT INTO knowledge_bases (id, storage_provider_config) VALUES + ('kb-pending', '{"provider":"__pending_env__"}'), + ('kb-real', '{"provider":"local"}'), + ('kb-null', NULL) + `).Error) + + t.Setenv("STORAGE_TYPE", "minio") + resolveStorageProviderPending(db) + + type row struct { + ID string `gorm:"column:id"` + ConfigJSON string `gorm:"column:storage_provider_config"` + } + var rows []row + require.NoError(t, db.Raw(`SELECT id, storage_provider_config FROM knowledge_bases ORDER BY id`).Scan(&rows).Error) + + byID := make(map[string]string, len(rows)) + for _, r := range rows { + byID[r.ID] = r.ConfigJSON + } + assert.Equal(t, `{"provider":"minio"}`, byID["kb-pending"], + "sentinel row must be rewritten with the env STORAGE_TYPE") + assert.Equal(t, `{"provider":"local"}`, byID["kb-real"], + "non-sentinel row must be left untouched") + assert.Equal(t, "", byID["kb-null"], + "NULL config must remain NULL (no provider to resolve)") +} diff --git a/internal/database/dialect.go b/internal/database/dialect.go new file mode 100644 index 0000000000..74cf7e4b9d --- /dev/null +++ b/internal/database/dialect.go @@ -0,0 +1,96 @@ +package database + +import ( + "fmt" +) + +// CaseInsensitiveLike returns a SQL fragment that performs a +// case-insensitive pattern match on the given column, shaped for the +// active database dialect. +// +// - postgres: "col ILIKE ?" - preserved so the existing pg_trgm GIN +// indexes on the affected columns stay usable. Switching PG to +// LOWER() LIKE LOWER() would require expression-index rebuilds and +// could regress search performance. +// - every other dialect (mysql, sqlite, anything future): the portable +// "LOWER(col) LIKE LOWER(?)" form. +func CaseInsensitiveLike(dialectName, column, placeholder string) string { + if dialectName == "postgres" { + return column + " ILIKE " + placeholder + } + return "LOWER(" + column + ") LIKE LOWER(" + placeholder + ")" +} + +// JSONPathExpr returns a SQL fragment that extracts a scalar value from +// a JSON column at the given key, shaped for the active database dialect. +// +// - postgres: "col ->> 'key'" (jsonb text extraction) +// - mysql: "col ->> '$.key'" (MySQL 8.0+ JSON path) +// - sqlite: "json_extract(col, '$.key')" (portable across SQLite versions) +// +// key must be a simple identifier matching [a-zA-Z0-9_-]. Dotted paths +// and metacharacters are rejected with an error rather than silently +// stripped — silently stripping turned "a.b" into "ab", which queried +// a different key than the caller intended. +func JSONPathExpr(dialectName, column, key string) (string, error) { + if err := validateJSONPathKey(key); err != nil { + return "", err + } + switch dialectName { + case "postgres": + return fmt.Sprintf("%s ->> '%s'", column, key), nil + case "mysql": + return fmt.Sprintf("%s ->> '$.%s'", column, key), nil + default: + return fmt.Sprintf("json_extract(%s, '$.%s')", column, key), nil + } +} + +// mysqlMaterializedJSONColumns maps a (JSON column, key) extraction to the +// generated column that materializes and indexes it in the MySQL schema. +// +// MySQL only substitutes an indexed generated column for equality-shaped +// predicates (=, <, IN, BETWEEN, ...), never for LIKE. A LIKE-prefix scan +// therefore has to name the generated column itself to stay on the index. +// Keeping the mapping here means the schema/query coupling is stated once, +// instead of a bare column name appearing at each call site where nothing +// would flag it if the migration renamed the column. +var mysqlMaterializedJSONColumns = map[[2]string]string{ + {"metadata", "external_id"}: "metadata_external_id", +} + +// JSONPathExprIndexed behaves like JSONPathExpr, except it returns the +// materialized generated column when the active dialect has one for this +// (column, key). Prefer it over JSONPathExpr wherever the predicate is +// expected to use an index. +func JSONPathExprIndexed(dialectName, column, key string) (string, error) { + if err := validateJSONPathKey(key); err != nil { + return "", err + } + if dialectName == "mysql" { + if generated, ok := mysqlMaterializedJSONColumns[[2]string{column, key}]; ok { + return generated, nil + } + } + return JSONPathExpr(dialectName, column, key) +} + +// validateJSONPathKey ensures key is a simple identifier ([a-zA-Z0-9_-]+). +// Empty keys and anything containing JSON path metacharacters (., [], *, +// quotes, etc.) are rejected. This prevents both injection and silent +// key corruption. +func validateJSONPathKey(key string) error { + if key == "" { + return fmt.Errorf("JSON path key must not be empty") + } + for _, r := range key { + if r == '_' || r == '-' || + (r >= 'a' && r <= 'z') || + (r >= 'A' && r <= 'Z') || + (r >= '0' && r <= '9') { + continue + } + return fmt.Errorf("JSON path key %q contains invalid character %q; only [a-zA-Z0-9_-] are allowed", key, string(r)) + } + return nil +} diff --git a/internal/database/dialect_test.go b/internal/database/dialect_test.go new file mode 100644 index 0000000000..4c3eb0558c --- /dev/null +++ b/internal/database/dialect_test.go @@ -0,0 +1,167 @@ +package database + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// CaseInsensitiveLike returns a SQL fragment that performs a +// case-insensitive pattern match on the given column, shaped for the +// active database dialect. +// +// The fragment choice encodes a real trade-off (see ADR 0001 S2): +// PostgreSQL has pg_trgm GIN indexes on the affected columns that +// ILIKE can hit; switching PG to LOWER() LIKE LOWER() would need +// expression-index rebuilds and could regress search performance. So +// postgres keeps ILIKE, while every other dialect uses the portable +// LOWER() LIKE LOWER() form. +// +// These tests are the contract for that helper. They are written +// before the helper exists (TDD red phase) and will not compile until +// internal/database/dialect.go is created. + +func TestCaseInsensitiveLike_Postgres_PreservesILIKE(t *testing.T) { + got := CaseInsensitiveLike("postgres", "title", "?") + assert.Equal(t, "title ILIKE ?", got, + "postgres must keep ILIKE so the existing pg_trgm GIN index stays usable") +} + +func TestCaseInsensitiveLike_MySQL_EmitsLowerLike(t *testing.T) { + got := CaseInsensitiveLike("mysql", "title", "?") + assert.Equal(t, "LOWER(title) LIKE LOWER(?)", got, + "mysql has no ILIKE; LOWER() LIKE LOWER() is the case-insensitive equivalent") +} + +func TestCaseInsensitiveLike_SQLite_EmitsLowerLike(t *testing.T) { + got := CaseInsensitiveLike("sqlite", "title", "?") + assert.Equal(t, "LOWER(title) LIKE LOWER(?)", got) +} + +func TestCaseInsensitiveLike_QualifiedColumnIsPreserved(t *testing.T) { + got := CaseInsensitiveLike("mysql", "messages.content", "?") + assert.Equal(t, "LOWER(messages.content) LIKE LOWER(?)", got, + "qualified columns (table.column) must pass through verbatim") +} + +// Unknown dialects fall back to the portable form. A future dialect +// that *does* have ILIKE would opt in explicitly; until then the +// safe default is the one every SQL engine understands. +func TestCaseInsensitiveLike_UnknownDialectFallsBackToLowerLike(t *testing.T) { + got := CaseInsensitiveLike("sqlserver", "title", "?") + assert.Equal(t, "LOWER(title) LIKE LOWER(?)", got, + "unknown dialects must fall back to the portable LOWER/LIKE form, not ILIKE") +} + +// JSONPathExpr returns a SQL fragment that extracts a scalar value from +// a JSON column. The fragment must match the dialect's native syntax so +// the caller's `= ?` comparison works; the bare-key postgres form +// errors on MySQL (1064 near '? = ?'), and the MySQL $.path form is not +// valid postgres syntax. +// +// JSONPathExpr now returns (string, error): it validates the key against +// [a-zA-Z0-9_-] first and returns an error on invalid input, rather than +// silently stripping characters. Silently stripping turned "a.b" into +// "ab", which queried a different key than the caller intended - a +// subtle correctness bug. Failing fast surfaces the bad caller instead. + +func TestJSONPathExpr_Postgres_UsesArrow(t *testing.T) { + got, err := JSONPathExpr("postgres", "metadata", "external_id") + assert.NoError(t, err) + assert.Equal(t, "metadata ->> 'external_id'", got) +} + +func TestJSONPathExpr_MySQL_UsesDollarPath(t *testing.T) { + got, err := JSONPathExpr("mysql", "metadata", "external_id") + assert.NoError(t, err) + assert.Equal(t, "metadata ->> '$.external_id'", got, + "mysql requires the $.key JSON path prefix; bare-key form errors with 1064") +} + +func TestJSONPathExpr_SQLite_UsesJsonExtract(t *testing.T) { + got, err := JSONPathExpr("sqlite", "metadata", "external_id") + assert.NoError(t, err) + assert.Equal(t, "json_extract(metadata, '$.external_id')", got) +} + +func TestJSONPathExpr_QualifiedColumnPreserved(t *testing.T) { + got, err := JSONPathExpr("mysql", "kb.metadata", "external_id") + assert.NoError(t, err) + assert.Equal(t, "kb.metadata ->> '$.external_id'", got) +} + +// validateJSONPathKey must reject anything outside [a-zA-Z0-9_-] or +// empty, so a caller-controlled key cannot inject a JSON path +// metacharacter (dot, bracket, quote, wildcard) AND cannot be silently +// rewritten into a different key than the caller intended. +func TestValidateJSONPathKey(t *testing.T) { + valid := []string{"external_id", "datasource_id", "source-resource-id", "key1", "A_B-C", "abc123"} + for _, k := range valid { + if err := validateJSONPathKey(k); err != nil { + t.Errorf("validateJSONPathKey(%q) returned unexpected error: %v", k, err) + } + } + invalid := []string{ + "", // empty + "a.b", // dot (was silently stripped to "ab" - the bug this guards) + "a[b]", // brackets + `a"b`, // quote + "a*b", // wildcard + "a b", // space + "a$b", // dollar + "中文", // non-ascii + } + for _, k := range invalid { + if err := validateJSONPathKey(k); err == nil { + t.Errorf("validateJSONPathKey(%q) should return an error, got nil", k) + } + } +} + +// JSONPathExpr must propagate the validation error for an invalid key +// rather than silently stripping and returning a fragment that targets +// the wrong JSON path. +func TestJSONPathExpr_InvalidKeyReturnsError(t *testing.T) { + got, err := JSONPathExpr("mysql", "metadata", "a.b") + assert.Error(t, err, "dotted key must error, not silently strip to 'ab'") + assert.Equal(t, "", got, "no fragment should be returned on validation error") +} + +func TestJSONPathExpr_EmptyKeyReturnsError(t *testing.T) { + got, err := JSONPathExpr("mysql", "metadata", "") + assert.Error(t, err, "empty key must error") + assert.Equal(t, "", got) +} + +// JSONPathExprIndexed swaps in the generated column that MySQL indexes, because +// MySQL only substitutes an indexed generated column for equality-shaped +// predicates and never for LIKE. PostgreSQL keeps the literal extraction +// expression, which is what its own expression index is built on. +func TestJSONPathExprIndexed_PrefersMaterializedColumnOnMySQL(t *testing.T) { + got, err := JSONPathExprIndexed("mysql", "metadata", "external_id") + assert.NoError(t, err) + assert.Equal(t, "metadata_external_id", got) + + got, err = JSONPathExprIndexed("postgres", "metadata", "external_id") + assert.NoError(t, err) + assert.Equal(t, "metadata ->> 'external_id'", got) + + got, err = JSONPathExprIndexed("sqlite", "metadata", "external_id") + assert.NoError(t, err) + assert.Equal(t, "json_extract(metadata, '$.external_id')", got) +} + +// A key with no materialized column must fall back to plain extraction on every +// dialect, so adding a new metadata key never silently queries a column that +// the migration does not define. +func TestJSONPathExprIndexed_FallsBackForUnmappedKeys(t *testing.T) { + got, err := JSONPathExprIndexed("mysql", "metadata", "some_other_key") + assert.NoError(t, err) + assert.Equal(t, "metadata ->> '$.some_other_key'", got) +} + +func TestJSONPathExprIndexed_InvalidKeyReturnsError(t *testing.T) { + got, err := JSONPathExprIndexed("mysql", "metadata", "a.b") + assert.Error(t, err) + assert.Equal(t, "", got) +} diff --git a/internal/database/migration.go b/internal/database/migration.go index 48620fad8f..611655366a 100644 --- a/internal/database/migration.go +++ b/internal/database/migration.go @@ -10,6 +10,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/golang-migrate/migrate/v4" + mysql3migrate "github.com/golang-migrate/migrate/v4/database/mysql" _ "github.com/golang-migrate/migrate/v4/database/postgres" sqlite3migrate "github.com/golang-migrate/migrate/v4/database/sqlite3" _ "github.com/golang-migrate/migrate/v4/source/file" @@ -78,6 +79,23 @@ func captureMigrationFailure(m *migrate.Migrate, err error) error { return err } +// migrationsPathForDSN picks the migration directory based on the DSN's +// scheme. SQLite and MySQL each ship a dialect-specific squash baseline. +// PostgreSQL keeps its original full history. +func migrationsPathForDSN(dsn string) string { + switch { + case strings.HasPrefix(dsn, "sqlite3://"): + return "file://migrations/sqlite" + case strings.HasPrefix(dsn, "mysql://"): + return "file://migrations/mysql" + default: + // postgres:// and any unrecognised scheme → the versioned + // PostgreSQL history. This preserves existing behaviour for + // every postgres deployment. + return "file://migrations/versioned" + } +} + // RunMigrations executes all pending database migrations // This should be called during application startup func RunMigrations(dsn string) error { @@ -90,11 +108,60 @@ type MigrationOptions struct { // by forcing to the previous version and retrying the migration AutoRecoverDirty bool + // FailOnDirty, when true, forces the migrator to surface an error on dirty + // state instead of attempting auto-recovery, regardless of AutoRecoverDirty. + // This is the fail-closed path for dialects whose DDL is not transactional + // (MySQL): forcing the version backward and re-running Up() can leave a + // half-applied schema that silently corrupts subsequent business queries, + // so recovery must be a human decision. Set by the container based on + // DB_DRIVER == "mysql". + FailOnDirty bool + // SQLiteDBPath is the raw filesystem path to the SQLite database file. // When set, the migrator opens the DB directly via sql.Open instead of // parsing a URL-based DSN, which avoids breakage when the path contains // spaces (e.g. macOS "Application Support"). SQLiteDBPath string + + // MySQLMigrationDSN is the go-sql-driver DSN already configured with + // multiStatements and any custom TLS registration. Using a database/sql + // instance avoids golang-migrate's URL parser trying to reconstruct custom + // TLS settings (which loses SNI and requires x-tls-* query parameters). + MySQLMigrationDSN string +} + +// dirtyStateErrorMessage returns the error shown when the migrator refuses +// to auto-recover from a dirty state. For MySQL (non-transactional DDL) it +// tells the operator to inspect manually; for PG/SQLite it suggests the +// force command. +func dirtyStateErrorMessage(version uint, isMySQL bool) string { + if isMySQL { + return fmt.Sprintf( + "database migration is in dirty state at version %d. "+ + "MySQL DDL is not transactional, so auto-recovery is disabled. "+ + "Inspect the partially-created schema manually, then force the "+ + "migration version backward or drop and recreate the database.", + version, + ) + } + forceVersion := int(version) - 1 + if version == 0 || forceVersion < 0 { + forceVersion = 0 + } + return fmt.Sprintf( + "database is in dirty state at version %d. This usually means a migration failed partway through. "+ + "To fix this:\n"+ + "1. Check if the migration partially applied changes and manually fix if needed\n"+ + "2. Use the force command to set the version to the last successful migration (usually %d):\n"+ + " ./scripts/migrate.sh force %d\n"+ + " Or if using make: make migrate-force version=%d\n"+ + "3. After fixing, restart the application to retry the migration\n"+ + "Or enable AutoRecoverDirty option to automatically retry", + version, + forceVersion, + forceVersion, + forceVersion, + ) } // RunMigrationsWithOptions executes all pending database migrations with custom options @@ -103,9 +170,9 @@ func RunMigrationsWithOptions(dsn string, opts MigrationOptions) error { logger.Infof(ctx, "Starting database migration...") - migrationsPath := "file://migrations/versioned" - if strings.HasPrefix(dsn, "sqlite3://") { - migrationsPath = "file://migrations/sqlite" + migrationsPath := migrationsPathForDSN(dsn) + if opts.MySQLMigrationDSN != "" { + migrationsPath = "file://migrations/mysql" } var m *migrate.Migrate @@ -132,6 +199,27 @@ func RunMigrationsWithOptions(dsn string, opts MigrationOptions) error { setMigrationState(0, false, wrapped.Error(), false) return wrapped } + } else if opts.MySQLMigrationDSN != "" { + sqlDB, err := sql.Open("mysql", opts.MySQLMigrationDSN) + if err != nil { + wrapped := fmt.Errorf("failed to open mysql db for migration: %w", err) + setMigrationState(0, false, wrapped.Error(), false) + return wrapped + } + driver, err := mysql3migrate.WithInstance(sqlDB, &mysql3migrate.Config{}) + if err != nil { + _ = sqlDB.Close() + wrapped := fmt.Errorf("failed to create mysql migrate driver: %w", err) + setMigrationState(0, false, wrapped.Error(), false) + return wrapped + } + m, err = migrate.NewWithDatabaseInstance(migrationsPath, "mysql", driver) + if err != nil { + _ = sqlDB.Close() + wrapped := fmt.Errorf("failed to create mysql migrate instance: %w", err) + setMigrationState(0, false, wrapped.Error(), false) + return wrapped + } } else { var err error m, err = migrate.New(migrationsPath, dsn) @@ -160,6 +248,13 @@ func RunMigrationsWithOptions(dsn string, opts MigrationOptions) error { // If database is in dirty state, try to recover or return error if oldDirty { logger.Warnf(ctx, "Database is in dirty state at version %d", oldVersion) + // FailOnDirty (MySQL) short-circuits: MySQL DDL is not transactional, + // so forcing the version backward and re-running Up() can leave a + // half-applied schema that silently corrupts business queries. Recovery + // must be a human decision, so surface an actionable error instead. + if opts.FailOnDirty { + return captureMigrationFailure(m, fmt.Errorf("%s", dirtyStateErrorMessage(oldVersion, true))) + } if opts.AutoRecoverDirty { logger.Infof(ctx, "AutoRecoverDirty is enabled, attempting recovery...") if err := recoverFromDirtyState(ctx, m, oldVersion); err != nil { @@ -168,25 +263,7 @@ func RunMigrationsWithOptions(dsn string, opts MigrationOptions) error { // Update oldVersion after recovery oldVersion, _, _ = m.Version() } else { - // Calculate the version to force to (usually the previous version) - forceVersion := int(oldVersion) - 1 - if oldVersion == 0 || forceVersion < 0 { - forceVersion = 0 - } - return captureMigrationFailure(m, fmt.Errorf( - "database is in dirty state at version %d. This usually means a migration failed partway through. "+ - "To fix this:\n"+ - "1. Check if the migration partially applied changes and manually fix if needed\n"+ - "2. Use the force command to set the version to the last successful migration (usually %d):\n"+ - " ./scripts/migrate.sh force %d\n"+ - " Or if using make: make migrate-force version=%d\n"+ - "3. After fixing, restart the application to retry the migration\n"+ - "Or enable AutoRecoverDirty option to automatically retry", - oldVersion, - forceVersion, - forceVersion, - forceVersion, - )) + return captureMigrationFailure(m, fmt.Errorf("%s", dirtyStateErrorMessage(oldVersion, false))) } } @@ -198,6 +275,16 @@ func RunMigrationsWithOptions(dsn string, opts MigrationOptions) error { currentVersion, currentDirty, versionCheckErr := m.Version() if versionCheckErr == nil && currentDirty { logger.Warnf(ctx, "Migration caused dirty state at version %d", currentVersion) + // FailOnDirty (MySQL) short-circuits here too: the migration went + // dirty mid-Up(), so forcing the version is unsafe for the same + // non-transactional-DDL reason as the pre-migration branch above. + if opts.FailOnDirty { + return captureMigrationFailure(m, fmt.Errorf( + "migration failed: %w; %s", + err, + dirtyStateErrorMessage(currentVersion, true), + )) + } if opts.AutoRecoverDirty { logger.Infof(ctx, "Attempting to recover from dirty state...") // Try to recover and retry diff --git a/internal/database/migrations_path_test.go b/internal/database/migrations_path_test.go new file mode 100644 index 0000000000..f91e1ff757 --- /dev/null +++ b/internal/database/migrations_path_test.go @@ -0,0 +1,71 @@ +package database + +import ( + "os" + "slices" + "testing" +) + +// migrationsPathForDSN picks the directory of migration .sql files +// based on the database DSN's scheme. +// +// - sqlite3:// → migrations/sqlite (squash baseline) +// - mysql:// → migrations/mysql (squash baseline) +// - anything else (postgres://, etc.) → migrations/versioned (the full +// PostgreSQL history) +func TestMigrationsPathForDSN(t *testing.T) { + tests := []struct { + name string + dsn string + want string + }{ + { + name: "sqlite dsn resolves to sqlite baseline dir", + dsn: "sqlite3:///tmp/test.db", + want: "file://migrations/sqlite", + }, + { + name: "mysql dsn resolves to mysql baseline dir", + dsn: "mysql://user:pass@tcp(host:3306)/db", + want: "file://migrations/mysql", + }, + { + name: "postgres dsn resolves to versioned dir (default)", + dsn: "postgres://user:pass@host:5432/db?sslmode=disable", + want: "file://migrations/versioned", + }, + { + name: "unknown scheme defaults to versioned dir", + dsn: "somethingelse://foo", + want: "file://migrations/versioned", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := migrationsPathForDSN(tt.dsn) + if got != tt.want { + t.Fatalf("migrationsPathForDSN(%q) = %q; want %q", tt.dsn, got, tt.want) + } + }) + } +} + +func TestMySQLMigrationsRemainSingleSquashedBaselinePair(t *testing.T) { + entries, err := os.ReadDir("../../migrations/mysql") + if err != nil { + t.Fatalf("read MySQL migration directory: %v", err) + } + + var files []string + for _, entry := range entries { + if !entry.IsDir() { + files = append(files, entry.Name()) + } + } + + want := []string{"000000_init.down.sql", "000000_init.up.sql"} + if !slices.Equal(files, want) { + t.Fatalf("MySQL migrations must remain the single squashed baseline pair; got %v, want %v", files, want) + } +} diff --git a/internal/database/mysqlconfig/config.go b/internal/database/mysqlconfig/config.go new file mode 100644 index 0000000000..7a4ded708c --- /dev/null +++ b/internal/database/mysqlconfig/config.go @@ -0,0 +1,271 @@ +package mysqlconfig + +import ( + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "os" + "strconv" + "strings" + "time" + + mysqlDriver "github.com/go-sql-driver/mysql" +) + +// PoolConfig holds validated MySQL connection-pool and timeout settings. +type PoolConfig struct { + MaxOpenConns int + MaxIdleConns int + ConnMaxLifetime time.Duration + ConnMaxIdleTime time.Duration +} + +// BuildDSN constructs separate go-sql-driver DSNs for application queries and +// migrations from the same validated environment. Only the migration DSN +// enables multiStatements because the squashed MySQL baseline contains the +// complete schema in one multi-statement file. +func BuildDSN(env func(string) string) ( + applicationDSN string, + migrationDSN string, + pool PoolConfig, + err error, +) { + host := strings.TrimSpace(env("DB_HOST")) + port := strings.TrimSpace(env("DB_PORT")) + if port == "" { + port = "3306" + } + user := strings.TrimSpace(env("DB_USER")) + password := env("DB_PASSWORD") + dbname := strings.TrimSpace(env("DB_NAME")) + + if host == "" { + return "", "", PoolConfig{}, fmt.Errorf("DB_HOST is empty; cannot construct MySQL DSN") + } + if user == "" { + return "", "", PoolConfig{}, fmt.Errorf("DB_USER is empty; cannot construct MySQL DSN") + } + if password == "" { + return "", "", PoolConfig{}, fmt.Errorf("DB_PASSWORD is empty; cannot construct MySQL DSN") + } + if dbname == "" { + return "", "", PoolConfig{}, fmt.Errorf("DB_NAME is empty; cannot construct MySQL DSN") + } + portNumber, err := strconv.Atoi(port) + if err != nil || portNumber < 1 || portNumber > 65535 { + return "", "", PoolConfig{}, fmt.Errorf("DB_PORT must be an integer between 1 and 65535, got %q", port) + } + + addr := net.JoinHostPort(host, port) + + cfg := mysqlDriver.NewConfig() + cfg.User = user + cfg.Passwd = password + cfg.Net = "tcp" + cfg.Addr = addr + cfg.DBName = dbname + cfg.Params = map[string]string{ + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "time_zone": "'+00:00'", + } + cfg.ParseTime = true + cfg.Loc = time.UTC + + cfg.Timeout, err = envDuration(env, "DB_CONNECT_TIMEOUT", 10*time.Second) + if err != nil { + return "", "", PoolConfig{}, err + } + cfg.ReadTimeout, err = envDuration(env, "DB_READ_TIMEOUT", 30*time.Second) + if err != nil { + return "", "", PoolConfig{}, err + } + cfg.WriteTimeout, err = envDuration(env, "DB_WRITE_TIMEOUT", 30*time.Second) + if err != nil { + return "", "", PoolConfig{}, err + } + + tlsConfigName, err := configureTLS(env) + if err != nil { + return "", "", PoolConfig{}, err + } + if tlsConfigName != "" { + cfg.TLSConfig = tlsConfigName + } + + applicationDSN = cfg.FormatDSN() + cfg.MultiStatements = true + migrationDSN = cfg.FormatDSN() + + maxOpen, err := envInt(env, "DB_MAX_OPEN_CONNS", 50, false) + if err != nil { + return "", "", PoolConfig{}, err + } + maxIdle, err := envInt(env, "DB_MAX_IDLE_CONNS", 10, true) + if err != nil { + return "", "", PoolConfig{}, err + } + if maxIdle > maxOpen { + return "", "", PoolConfig{}, fmt.Errorf( + "DB_MAX_IDLE_CONNS (%d) must not exceed DB_MAX_OPEN_CONNS (%d)", + maxIdle, + maxOpen, + ) + } + connMaxLifetime, err := envDuration(env, "DB_CONN_MAX_LIFETIME", 10*time.Minute) + if err != nil { + return "", "", PoolConfig{}, err + } + connMaxIdleTime, err := envDuration(env, "DB_CONN_MAX_IDLE_TIME", 5*time.Minute) + if err != nil { + return "", "", PoolConfig{}, err + } + + return applicationDSN, migrationDSN, PoolConfig{ + MaxOpenConns: maxOpen, + MaxIdleConns: maxIdle, + ConnMaxLifetime: connMaxLifetime, + ConnMaxIdleTime: connMaxIdleTime, + }, nil +} + +func envDuration( + env func(string) string, + key string, + defaultValue time.Duration, +) (time.Duration, error) { + value := strings.TrimSpace(env(key)) + if value == "" { + return defaultValue, nil + } + duration, err := time.ParseDuration(value) + if err != nil { + return 0, fmt.Errorf("%s must be a valid duration, got %q: %w", key, value, err) + } + if duration <= 0 { + return 0, fmt.Errorf("%s must be greater than zero, got %q", key, value) + } + return duration, nil +} + +func envInt( + env func(string) string, + key string, + defaultValue int, + allowZero bool, +) (int, error) { + value := strings.TrimSpace(env(key)) + if value == "" { + return defaultValue, nil + } + number, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("%s must be an integer, got %q: %w", key, value, err) + } + if number < 0 || (!allowZero && number == 0) { + requirement := "greater than zero" + if allowZero { + requirement = "zero or greater" + } + return 0, fmt.Errorf("%s must be %s, got %d", key, requirement, number) + } + return number, nil +} + +func envBool(env func(string) string, key string, defaultValue bool) (bool, error) { + value := strings.TrimSpace(env(key)) + if value == "" { + return defaultValue, nil + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return false, fmt.Errorf("%s must be true or false, got %q: %w", key, value, err) + } + return parsed, nil +} + +func configureTLS(env func(string) string) (string, error) { + useTLS, err := envBool(env, "DB_USE_TLS", false) + if err != nil { + return "", err + } + insecureSkipVerify, err := envBool(env, "DB_TLS_INSECURE_SKIP_VERIFY", false) + if err != nil { + return "", err + } + + serverName := strings.TrimSpace(env("DB_TLS_SERVER_NAME")) + caFile := strings.TrimSpace(env("DB_TLS_CA")) + certFile := strings.TrimSpace(env("DB_TLS_CERT")) + keyFile := strings.TrimSpace(env("DB_TLS_KEY")) + hasCustomSettings := serverName != "" || + caFile != "" || + certFile != "" || + keyFile != "" || + insecureSkipVerify + if !useTLS { + if hasCustomSettings { + return "", fmt.Errorf("DB_USE_TLS must be true when DB_TLS_* settings are configured") + } + return "", nil + } + + if (certFile == "") != (keyFile == "") { + if certFile != "" { + return "", fmt.Errorf("DB_TLS_CERT requires DB_TLS_KEY") + } + return "", fmt.Errorf("DB_TLS_KEY requires DB_TLS_CERT") + } + if !hasCustomSettings { + return "true", nil + } + + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + ServerName: serverName, + InsecureSkipVerify: insecureSkipVerify, //nolint:gosec // Explicit development-only operator setting. + } + + fingerprint := sha256.New() + _, _ = fingerprint.Write([]byte(serverName)) + _, _ = fingerprint.Write([]byte(strconv.FormatBool(insecureSkipVerify))) + + if caFile != "" { + caPEM, readErr := os.ReadFile(caFile) + if readErr != nil { + return "", fmt.Errorf("read DB_TLS_CA %q: %w", caFile, readErr) + } + rootCAs, poolErr := x509.SystemCertPool() + if poolErr != nil || rootCAs == nil { + rootCAs = x509.NewCertPool() + } + if !rootCAs.AppendCertsFromPEM(caPEM) { + return "", fmt.Errorf("DB_TLS_CA %q does not contain a valid PEM certificate", caFile) + } + tlsConfig.RootCAs = rootCAs + _, _ = fingerprint.Write(caPEM) + } + + if certFile != "" { + certificate, loadErr := tls.LoadX509KeyPair(certFile, keyFile) + if loadErr != nil { + return "", fmt.Errorf( + "load DB_TLS_CERT %q and DB_TLS_KEY %q: %w", + certFile, + keyFile, + loadErr, + ) + } + tlsConfig.Certificates = []tls.Certificate{certificate} + _, _ = fingerprint.Write([]byte(certFile)) + _, _ = fingerprint.Write([]byte(keyFile)) + } + + configName := fmt.Sprintf("weknora-%x", fingerprint.Sum(nil)[:8]) + if err := mysqlDriver.RegisterTLSConfig(configName, tlsConfig); err != nil { + return "", fmt.Errorf("register MySQL TLS configuration: %w", err) + } + return configName, nil +} diff --git a/internal/database/mysqlconfig/config_test.go b/internal/database/mysqlconfig/config_test.go new file mode 100644 index 0000000000..d80a25252e --- /dev/null +++ b/internal/database/mysqlconfig/config_test.go @@ -0,0 +1,316 @@ +package mysqlconfig + +import ( + "strings" + "testing" + "time" + + mysqlDriver "github.com/go-sql-driver/mysql" +) + +func testEnv(vals map[string]string) func(string) string { + return func(key string) string { return vals[key] } +} + +func baseEnv() map[string]string { + return map[string]string{ + "DB_HOST": "127.0.0.1", + "DB_PORT": "3306", + "DB_USER": "weknora", + "DB_PASSWORD": "secret", + "DB_NAME": "weknora_db", + } +} + +func mustParseMySQLDSN(t *testing.T, dsn string) *mysqlDriver.Config { + t.Helper() + config, err := mysqlDriver.ParseDSN(dsn) + if err != nil { + t.Fatalf("parse MySQL DSN: %v", err) + } + return config +} + +func TestBuildDSN_BasicShape(t *testing.T) { + gormDSN, migrateDSN, pool, err := BuildDSN(testEnv(baseEnv())) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for _, want := range []string{ + "weknora", + "secret", + "tcp(127.0.0.1:3306)", + "weknora_db", + "charset=utf8mb4", + "parseTime=true", + } { + if !strings.Contains(gormDSN, want) { + t.Errorf("gormDSN missing %q; got: %s", want, gormDSN) + } + } + applicationConfig := mustParseMySQLDSN(t, gormDSN) + migrationConfig := mustParseMySQLDSN(t, migrateDSN) + if applicationConfig.MultiStatements { + t.Errorf("application DSN must keep multiStatements disabled; got: %s", gormDSN) + } + if !migrationConfig.MultiStatements { + t.Errorf("migration DSN must enable multiStatements; got: %s", migrateDSN) + } + if migrationConfig.Addr != applicationConfig.Addr || + migrationConfig.User != applicationConfig.User || + migrationConfig.DBName != applicationConfig.DBName || + migrationConfig.Loc != applicationConfig.Loc { + t.Fatal("deriving the migration DSN changed application connection settings") + } + if pool.MaxOpenConns != 50 || pool.MaxIdleConns != 10 { + t.Errorf("pool defaults wrong: %+v", pool) + } +} + +func TestBuildDSN_CollationIsPinned(t *testing.T) { + gormDSN, migrateDSN, _, err := BuildDSN(testEnv(baseEnv())) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.Contains(gormDSN, "collation=utf8mb4_0900_ai_ci") { + t.Errorf("gormDSN must pin collation; got: %s", gormDSN) + } + if !strings.Contains(migrateDSN, "collation=utf8mb4_0900_ai_ci") { + t.Errorf("migrateDSN must pin collation; got: %s", migrateDSN) + } +} + +func TestBuildDSN_TLSUsesVerifiedTransport(t *testing.T) { + env := baseEnv() + env["DB_USE_TLS"] = "true" + + gormDSN, migrateDSN, _, err := BuildDSN(testEnv(env)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + for name, dsn := range map[string]string{ + "application": gormDSN, + "migration": migrateDSN, + } { + if !strings.Contains(dsn, "tls=true") { + t.Errorf("%s DSN must require verified TLS; got: %s", name, dsn) + } + } +} + +func TestBuildDSN_TLSCustomVerificationSettings(t *testing.T) { + env := baseEnv() + env["DB_USE_TLS"] = "true" + env["DB_TLS_SERVER_NAME"] = "mysql.internal.example" + env["DB_TLS_INSECURE_SKIP_VERIFY"] = "true" + + gormDSN, migrateDSN, _, err := BuildDSN(testEnv(env)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + for name, dsn := range map[string]string{ + "application": gormDSN, + "migration": migrateDSN, + } { + if !strings.Contains(dsn, "tls=weknora-") { + t.Errorf("%s DSN must reference the registered custom TLS config; got: %s", name, dsn) + } + } +} + +func TestBuildDSN_InvalidTLSConfigurationErrors(t *testing.T) { + tests := []struct { + name string + values map[string]string + errContains string + }{ + { + name: "invalid use tls boolean", + values: map[string]string{"DB_USE_TLS": "sometimes"}, + errContains: "DB_USE_TLS", + }, + { + name: "custom settings require tls", + values: map[string]string{ + "DB_USE_TLS": "false", + "DB_TLS_SERVER_NAME": "mysql.example", + }, + errContains: "DB_USE_TLS", + }, + { + name: "client certificate requires key", + values: map[string]string{ + "DB_USE_TLS": "true", + "DB_TLS_CERT": "/tmp/client.pem", + }, + errContains: "DB_TLS_CERT", + }, + { + name: "client key requires certificate", + values: map[string]string{ + "DB_USE_TLS": "true", + "DB_TLS_KEY": "/tmp/client-key.pem", + }, + errContains: "DB_TLS_KEY", + }, + { + name: "missing ca file", + values: map[string]string{ + "DB_USE_TLS": "true", + "DB_TLS_CA": "/definitely/missing/mysql-ca.pem", + }, + errContains: "DB_TLS_CA", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + env := baseEnv() + for key, value := range tt.values { + env[key] = value + } + + _, _, _, err := BuildDSN(testEnv(env)) + if err == nil { + t.Fatal("expected TLS configuration error") + } + if !strings.Contains(err.Error(), tt.errContains) { + t.Fatalf("error %q must mention %s", err, tt.errContains) + } + }) + } +} + +func TestBuildDSN_IPv6AddressWrapped(t *testing.T) { + env := baseEnv() + env["DB_HOST"] = "::1" + gormDSN, _, _, err := BuildDSN(testEnv(env)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // IPv6 host must be wrapped in [...] + if !strings.Contains(gormDSN, "tcp([::1]:3306)") { + t.Errorf("IPv6 host must be bracketed; got: %s", gormDSN) + } +} + +func TestBuildDSN_LocIsUTC(t *testing.T) { + applicationDSN, migrationDSN, _, err := BuildDSN(testEnv(baseEnv())) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + for name, dsn := range map[string]string{ + "application": applicationDSN, + "migration": migrationDSN, + } { + if config := mustParseMySQLDSN(t, dsn); config.Loc != time.UTC { + t.Errorf("%s DSN must decode timestamps in UTC; got location %s", name, config.Loc) + } + } +} + +func TestBuildDSN_SessionTimeZoneIsUTC(t *testing.T) { + gormDSN, migrateDSN, _, err := BuildDSN(testEnv(baseEnv())) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + for name, dsn := range map[string]string{ + "application": gormDSN, + "migration": migrateDSN, + } { + if !strings.Contains(dsn, "time_zone=%27%2B00%3A00%27") { + t.Errorf("%s DSN must set the MySQL session time_zone to UTC; got: %s", name, dsn) + } + } +} + +func TestBuildDSN_EmptyHostErrors(t *testing.T) { + env := baseEnv() + env["DB_HOST"] = "" + _, _, _, err := BuildDSN(testEnv(env)) + if err == nil { + t.Fatal("expected error for empty DB_HOST") + } +} + +func TestBuildDSN_InvalidConfigurationErrors(t *testing.T) { + tests := []struct { + name string + key string + value string + errContains string + }{ + {name: "empty user", key: "DB_USER", value: "", errContains: "DB_USER"}, + {name: "empty password", key: "DB_PASSWORD", value: "", errContains: "DB_PASSWORD"}, + {name: "empty database", key: "DB_NAME", value: "", errContains: "DB_NAME"}, + {name: "non numeric port", key: "DB_PORT", value: "mysql", errContains: "DB_PORT"}, + {name: "port out of range", key: "DB_PORT", value: "65536", errContains: "DB_PORT"}, + {name: "invalid connect timeout", key: "DB_CONNECT_TIMEOUT", value: "soon", errContains: "DB_CONNECT_TIMEOUT"}, + {name: "zero read timeout", key: "DB_READ_TIMEOUT", value: "0s", errContains: "DB_READ_TIMEOUT"}, + {name: "negative write timeout", key: "DB_WRITE_TIMEOUT", value: "-1s", errContains: "DB_WRITE_TIMEOUT"}, + {name: "invalid max open", key: "DB_MAX_OPEN_CONNS", value: "many", errContains: "DB_MAX_OPEN_CONNS"}, + {name: "zero max open", key: "DB_MAX_OPEN_CONNS", value: "0", errContains: "DB_MAX_OPEN_CONNS"}, + {name: "negative max idle", key: "DB_MAX_IDLE_CONNS", value: "-1", errContains: "DB_MAX_IDLE_CONNS"}, + {name: "invalid max lifetime", key: "DB_CONN_MAX_LIFETIME", value: "later", errContains: "DB_CONN_MAX_LIFETIME"}, + {name: "zero max idle time", key: "DB_CONN_MAX_IDLE_TIME", value: "0", errContains: "DB_CONN_MAX_IDLE_TIME"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + env := baseEnv() + env[tt.key] = tt.value + + _, _, _, err := BuildDSN(testEnv(env)) + if err == nil { + t.Fatalf("expected an error for %s=%q", tt.key, tt.value) + } + if !strings.Contains(err.Error(), tt.errContains) { + t.Fatalf("error %q must name %s", err, tt.errContains) + } + }) + } +} + +func TestBuildDSN_DefaultPortWhenUnset(t *testing.T) { + env := baseEnv() + delete(env, "DB_PORT") + gormDSN, _, _, err := BuildDSN(testEnv(env)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.Contains(gormDSN, "tcp(127.0.0.1:3306)") { + t.Errorf("gormDSN must default to port 3306; got: %s", gormDSN) + } +} + +func TestBuildDSN_PreservesPasswordWithSpecialCharacters(t *testing.T) { + env := baseEnv() + env["DB_PASSWORD"] = "p@ss/w:ord" + gormDSN, migrateDSN, _, err := BuildDSN(testEnv(env)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + for name, dsn := range map[string]string{ + "application": gormDSN, + "migration": migrateDSN, + } { + if password := mustParseMySQLDSN(t, dsn).Passwd; password != env["DB_PASSWORD"] { + t.Errorf("%s DSN password changed: got %q, want %q", name, password, env["DB_PASSWORD"]) + } + } +} + +func TestBuildDSN_MaxIdleExceedsMaxOpenErrors(t *testing.T) { + env := baseEnv() + env["DB_MAX_OPEN_CONNS"] = "5" + env["DB_MAX_IDLE_CONNS"] = "10" + _, _, _, err := BuildDSN(testEnv(env)) + if err == nil { + t.Fatal("expected error when maxIdle > maxOpen") + } +} diff --git a/internal/types/builtin_models_config.go b/internal/types/builtin_models_config.go index 7f1d508e5e..7d9e4ebc97 100644 --- a/internal/types/builtin_models_config.go +++ b/internal/types/builtin_models_config.go @@ -246,7 +246,7 @@ var validBuiltinModelStatuses = map[ModelStatus]struct{}{ // either crash the INSERT or silently produce an unusable row: // // - empty id (cannot UPSERT) -// - id longer than the DB column (PG/SQLite cap at varchar(64), +// - id longer than the DB column (PG/SQLite/MySQL cap at varchar(64), // see ModelIDMaxLen) which would fail at INSERT time // - empty or misspelled type (provider factories match exact strings) // - explicit non-empty status outside the known set diff --git a/internal/types/interfaces/task_queue.go b/internal/types/interfaces/task_queue.go index 7eb2005cb1..02beb9f583 100644 --- a/internal/types/interfaces/task_queue.go +++ b/internal/types/interfaces/task_queue.go @@ -19,7 +19,7 @@ import ( // serialization out-of-band (e.g. an external Redis lock). This is the // original primitive and is still used by single-process (Lite) mode. // - ClaimBatch DOES take row locks (SELECT ... FOR UPDATE SKIP LOCKED on -// Postgres) and marks rows with claimed_at, so multiple concurrent +// PostgreSQL/MySQL) and marks rows with claimed_at, so multiple concurrent // consumers of the same tuple pull DISJOINT rows without an external // lock. This is what lets the wiki pipeline drop its exclusive per-KB // batch lock and spread one KB's backlog across the whole worker pool. @@ -42,8 +42,8 @@ type TaskPendingOpsRepository interface { // several queued ops is never split across two concurrent batches. // A row is eligible when it is unclaimed (claimed_at IS NULL) or its // claim is stale (claimed_at < staleBefore) — the latter recovers rows - // abandoned by a crashed worker. On Postgres the per-key anchor row is - // locked with FOR UPDATE SKIP LOCKED so concurrent claimers take + // abandoned by a crashed worker. On PostgreSQL and MySQL 8 the per-key + // anchor row is locked with FOR UPDATE SKIP LOCKED so concurrent claimers take // disjoint key sets without blocking or double-claiming. // // Claimed rows are NOT removed: the consumer must DeleteByIDs on diff --git a/internal/types/interfaces/wiki_page.go b/internal/types/interfaces/wiki_page.go index 9d2c9662c6..7bc4c61557 100644 --- a/internal/types/interfaces/wiki_page.go +++ b/internal/types/interfaces/wiki_page.go @@ -148,9 +148,9 @@ type WikiPageService interface { // Used by rebuildIndexPage's first-time generation path. ListByTypeRecent(ctx context.Context, kbID string, pageType string, limit int) ([]types.WikiIndexEntry, error) - // FindSimilarPages performs a pg_trgm similarity search over - // page titles. Used by the dedup pre-filter to surface candidate - // merge targets server-side. + // FindSimilarPages performs dialect-aware candidate matching over page + // titles. Used by the dedup pre-filter to surface merge targets + // server-side. FindSimilarPages(ctx context.Context, kbID string, query string, pageTypes []string, limit int) ([]*types.WikiPageLite, error) // ListDistinctCategoryPaths returns the existing wiki folder paths (split @@ -312,10 +312,10 @@ type WikiPageRepository interface { // LLM prompt size is bounded on large KBs. ListByTypeRecent(ctx context.Context, kbID string, pageType string, limit int) ([]types.WikiIndexEntry, error) - // FindSimilarPages returns the top-k pages whose lowercase title - // is most similar to the query under pg_trgm. `pageTypes` empty - // defaults to entity+concept. Used by the dedup pre-filter to - // surface candidate merge targets server-side. + // FindSimilarPages returns the top-k pages whose lowercase title matches + // the query under the active database dialect. `pageTypes` empty defaults + // to entity+concept. Used by the dedup pre-filter to surface candidate + // merge targets server-side. FindSimilarPages(ctx context.Context, kbID string, query string, pageTypes []string, limit int) ([]*types.WikiPageLite, error) // ListDistinctCategoryPaths returns the materialized paths of existing diff --git a/internal/types/mcp_oauth.go b/internal/types/mcp_oauth.go index 696b91347f..25f32d64db 100644 --- a/internal/types/mcp_oauth.go +++ b/internal/types/mcp_oauth.go @@ -84,7 +84,7 @@ type MCPOAuthToken struct { AccessToken string `json:"-" gorm:"type:text"` RefreshToken string `json:"-" gorm:"type:text"` TokenType string `json:"token_type" gorm:"type:varchar(32)"` - ExpiresAt time.Time `json:"expires_at"` + ExpiresAt time.Time `json:"expires_at" gorm:"default:null"` // RefreshLeaseID / RefreshLeaseUntil coordinate refresh-token rotation // across application instances. They are operational fields only and are // never exposed through the API. diff --git a/internal/types/model.go b/internal/types/model.go index 2af9828384..532a9d0ac6 100644 --- a/internal/types/model.go +++ b/internal/types/model.go @@ -94,11 +94,10 @@ type ModelParameters struct { // (mutates an entity that other code may still be using). // ModelIDMaxLen is the upper bound on `models.id`. Matches the actual -// schema width on both PostgreSQL (varchar(64) in migrations/versioned/ -// 000000_init.up.sql) and SQLite (varchar(64) in migrations/sqlite/ -// 000000_init.up.sql). Loaders that accept user-provided ids (e.g. the -// built-in models YAML loader) must reject anything longer to avoid a -// "value too long for type" failure at INSERT time. +// schema width on PostgreSQL, SQLite, and MySQL. Loaders that accept +// user-provided ids (e.g. the built-in models YAML loader) must reject +// anything longer to avoid a "value too long for type" failure at INSERT +// time. const ModelIDMaxLen = 64 // DefaultBuiltinModelTenantID is the tenant id that built-in models are @@ -111,7 +110,7 @@ const DefaultBuiltinModelTenantID uint64 = 10000 // Model represents the AI model type Model struct { // Unique identifier of the model. The actual DB schema width is - // varchar(64) on both PostgreSQL and SQLite (see ModelIDMaxLen); + // varchar(64) on PostgreSQL, SQLite, and MySQL (see ModelIDMaxLen); // GORM's struct tag is documented to match so AutoMigrate paths // produce the same shape. ID string `yaml:"id" json:"id" gorm:"type:varchar(64);primaryKey"` diff --git a/internal/types/system_setting.go b/internal/types/system_setting.go index 838cfe5d2e..09fec4af35 100644 --- a/internal/types/system_setting.go +++ b/internal/types/system_setting.go @@ -24,7 +24,7 @@ import ( // roundtrip as `true`/`false`, ints as `42`, strings as `"foo"`. type SystemSetting struct { ID uint64 `gorm:"primaryKey" json:"id"` - Key string `gorm:"type:varchar(128);uniqueIndex;not null" json:"key"` + Key string `gorm:"column:key;type:varchar(128);uniqueIndex;not null" json:"key"` Value JSON `gorm:"type:jsonb;not null" json:"value"` // ValueType is one of "int", "string", "bool". Service layer rejects // updates whose payload type does not match; UI uses it to pick diff --git a/internal/types/wiki_page.go b/internal/types/wiki_page.go index e9035e2398..ad393cbec8 100644 --- a/internal/types/wiki_page.go +++ b/internal/types/wiki_page.go @@ -173,7 +173,7 @@ type WikiPage struct { KnowledgeBaseID string `json:"knowledge_base_id" gorm:"type:varchar(36);index"` // URL-friendly slug for addressing, e.g. "entity/acme-corp", "concept/rag" // Unique within a knowledge base - Slug string `json:"slug" gorm:"type:varchar(255);uniqueIndex:idx_kb_slug"` + Slug string `json:"slug" gorm:"type:varchar(255);index:idx_wiki_pages_kb_slug_live"` // Human-readable title Title string `json:"title" gorm:"type:varchar(512)"` // Page type: summary, entity, concept, index, synthesis, comparison diff --git a/migrations/mysql/00-init-db.sql b/migrations/mysql/00-init-db.sql deleted file mode 100644 index a4c7aec608..0000000000 --- a/migrations/mysql/00-init-db.sql +++ /dev/null @@ -1,229 +0,0 @@ -DROP TABLE IF EXISTS tenants; -DROP TABLE IF EXISTS models; -DROP TABLE IF EXISTS knowledge_bases; -DROP TABLE IF EXISTS knowledges; -DROP TABLE IF EXISTS sessions; -DROP TABLE IF EXISTS messages; -DROP TABLE IF EXISTS chunks; - -CREATE TABLE tenants ( - id BIGINT AUTO_INCREMENT PRIMARY KEY, - name VARCHAR(255) NOT NULL, - description TEXT, - retriever_engines JSON NOT NULL, - status VARCHAR(50) DEFAULT 'active', - business VARCHAR(255) NOT NULL, - storage_quota BIGINT NOT NULL DEFAULT 10737418240, - storage_used BIGINT NOT NULL DEFAULT 0, - agent_config JSON DEFAULT NULL COMMENT 'Tenant-level agent configuration in JSON format', - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 AUTO_INCREMENT=10000; - -CREATE TABLE models ( - id VARCHAR(64) PRIMARY KEY, - tenant_id INT NOT NULL, - name VARCHAR(255) NOT NULL, - display_name VARCHAR(255) NOT NULL DEFAULT '', - type VARCHAR(50) NOT NULL, - source VARCHAR(50) NOT NULL, - description TEXT, - parameters JSON NOT NULL, - is_default BOOLEAN NOT NULL DEFAULT FALSE, - status VARCHAR(50) NOT NULL DEFAULT 'active', - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE INDEX idx_models_tenant_source_type ON models(tenant_id, source, type); - -CREATE TABLE knowledge_bases ( - id VARCHAR(36) PRIMARY KEY, - name VARCHAR(255) NOT NULL, - description TEXT, - tenant_id INT NOT NULL, - chunking_config JSON NOT NULL, - image_processing_config JSON NOT NULL, - embedding_model_id VARCHAR(64) NOT NULL, - summary_model_id VARCHAR(64) NOT NULL, - rerank_model_id VARCHAR(64) NOT NULL, - cos_config JSON NOT NULL, - vlm_config JSON NOT NULL, - extract_config JSON NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE INDEX idx_knowledge_bases_tenant_name ON knowledge_bases(tenant_id, name); - -CREATE TABLE knowledges ( - id VARCHAR(36) PRIMARY KEY, - tenant_id INT NOT NULL, - knowledge_base_id VARCHAR(36) NOT NULL, - type VARCHAR(50) NOT NULL, - title VARCHAR(255) NOT NULL, - description TEXT, - source VARCHAR(2048) NOT NULL, - parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed', - enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled', - embedding_model_id VARCHAR(64), - file_name VARCHAR(255), - file_type VARCHAR(50), - file_size BIGINT, - file_path TEXT, - file_hash VARCHAR(64), - storage_size BIGINT NOT NULL DEFAULT 0, - metadata JSON, - custom_metadata JSON NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL, - processed_at TIMESTAMP, - error_message TEXT -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE INDEX idx_knowledges_tenant_id ON knowledges(tenant_id, knowledge_base_id); - -CREATE TABLE sessions ( - id VARCHAR(36) PRIMARY KEY, - tenant_id INTEGER NOT NULL, - title VARCHAR(255), - description TEXT, - knowledge_base_id VARCHAR(36), - max_rounds INT NOT NULL DEFAULT 5, - enable_rewrite BOOLEAN NOT NULL DEFAULT TRUE, - fallback_strategy VARCHAR(255) NOT NULL DEFAULT 'fixed', - fallback_response VARCHAR(255) NOT NULL DEFAULT '很抱歉，我暂时无法回答这个问题。', - keyword_threshold FLOAT NOT NULL DEFAULT 0.5, - vector_threshold FLOAT NOT NULL DEFAULT 0.5, - rerank_model_id VARCHAR(64), - embedding_top_k INTEGER NOT NULL DEFAULT 10, - rerank_top_k INTEGER NOT NULL DEFAULT 10, - rerank_threshold FLOAT NOT NULL DEFAULT 0.65, - summary_model_id VARCHAR(64), - summary_parameters JSON NOT NULL, - agent_config JSON DEFAULT NULL COMMENT 'Session-level agent configuration in JSON format', - context_config JSON DEFAULT NULL COMMENT 'LLM context management configuration (separate from message storage)', - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE INDEX idx_sessions_tenant_id ON sessions(tenant_id); - -CREATE TABLE messages ( - id VARCHAR(36) PRIMARY KEY, - request_id VARCHAR(36) NOT NULL, - session_id VARCHAR(36) NOT NULL, - role VARCHAR(50) NOT NULL, - content TEXT NOT NULL, - knowledge_references JSON NOT NULL, - agent_steps JSON DEFAULT NULL COMMENT 'Agent execution steps (reasoning process and tool calls)', - is_completed BOOLEAN NOT NULL DEFAULT FALSE, - agent_id VARCHAR(36) NOT NULL DEFAULT '', - agent_tenant_id INTEGER NOT NULL DEFAULT 0, - model_id VARCHAR(64) NOT NULL DEFAULT '', - execution_context JSON NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE INDEX idx_messages_session_role ON messages(session_id, role); -CREATE INDEX idx_messages_agent_id ON messages(agent_id); - -CREATE TABLE message_suggestion_sets ( - id VARCHAR(36) PRIMARY KEY, - tenant_id INTEGER NOT NULL, - session_id VARCHAR(36) NOT NULL, - assistant_message_id VARCHAR(36) NOT NULL, - agent_id VARCHAR(36) NOT NULL DEFAULT '', - agent_tenant_id INTEGER NOT NULL DEFAULT 0, - placement VARCHAR(32) NOT NULL, - config_hash VARCHAR(64) NOT NULL, - locale VARCHAR(16) NOT NULL DEFAULT '', - status VARCHAR(16) NOT NULL, - allow_regenerate BOOLEAN NOT NULL DEFAULT FALSE, - suppression_reason VARCHAR(64) NOT NULL DEFAULT '', - questions JSON NOT NULL, - model_id VARCHAR(64) NOT NULL DEFAULT '', - prompt_tokens INTEGER NOT NULL DEFAULT 0, - completion_tokens INTEGER NOT NULL DEFAULT 0, - latency_ms BIGINT NOT NULL DEFAULT 0, - error_code VARCHAR(64) NOT NULL DEFAULT '', - lease_until TIMESTAMP NULL DEFAULT NULL, - generated_at TIMESTAMP NULL DEFAULT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - UNIQUE KEY idx_message_suggestion_sets_cache_key - (tenant_id, assistant_message_id, placement, config_hash, locale), - KEY idx_message_suggestion_sets_session (tenant_id, session_id, created_at), - KEY idx_message_suggestion_sets_status (status, lease_until) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE message_suggestion_events ( - id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, - tenant_id INTEGER NOT NULL, - session_id VARCHAR(36) NOT NULL, - suggestion_set_id VARCHAR(36) NOT NULL, - question_id VARCHAR(64) NOT NULL DEFAULT '', - event_type VARCHAR(32) NOT NULL, - actor_id VARCHAR(512) NOT NULL DEFAULT '', - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - KEY idx_message_suggestion_events_set (suggestion_set_id, created_at), - KEY idx_message_suggestion_events_session (tenant_id, session_id, created_at), - KEY idx_message_suggestion_events_type (event_type, created_at), - CONSTRAINT fk_message_suggestion_events_set - FOREIGN KEY (suggestion_set_id) REFERENCES message_suggestion_sets(id) ON DELETE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE TABLE chunks ( - id VARCHAR(36) PRIMARY KEY, - tenant_id INTEGER NOT NULL, - knowledge_base_id VARCHAR(36) NOT NULL, - knowledge_id VARCHAR(36) NOT NULL, - content TEXT NOT NULL, - source_content TEXT NOT NULL, - content_revision INT NOT NULL DEFAULT 0, - index_status VARCHAR(16) NOT NULL DEFAULT 'ready', - last_editor_id VARCHAR(64) NOT NULL DEFAULT '', - context_header TEXT NOT NULL, - chunk_index INTEGER NOT NULL, - is_enabled BOOLEAN NOT NULL DEFAULT TRUE, - start_at INTEGER NOT NULL, - end_at INTEGER NOT NULL, - pre_chunk_id VARCHAR(36), - next_chunk_id VARCHAR(36), - chunk_type VARCHAR(20) NOT NULL DEFAULT 'text', - parent_chunk_id VARCHAR(36), - image_info TEXT, - relation_chunks JSON, - indirect_relation_chunks JSON, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - deleted_at TIMESTAMP NULL DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -CREATE INDEX idx_chunks_tenant_knowledge ON chunks(tenant_id, knowledge_id); -CREATE INDEX idx_chunks_parent_id ON chunks(parent_chunk_id); -CREATE INDEX idx_chunks_chunk_type ON chunks(chunk_type); - -CREATE TABLE chunk_revisions ( - id VARCHAR(36) PRIMARY KEY, - tenant_id BIGINT NOT NULL, - knowledge_base_id VARCHAR(36) NOT NULL, - knowledge_id VARCHAR(36) NOT NULL, - chunk_id VARCHAR(36) NOT NULL, - revision INT NOT NULL, - content TEXT NOT NULL, - is_enabled BOOLEAN NOT NULL DEFAULT TRUE, - editor_id VARCHAR(64) NOT NULL DEFAULT '', - edit_source VARCHAR(16) NOT NULL DEFAULT 'user', - edited_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - UNIQUE KEY idx_chunk_revisions_chunk_revision (chunk_id, revision), - KEY idx_chunk_revisions_tenant_chunk (tenant_id, chunk_id) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/migrations/mysql/000000_init.down.sql b/migrations/mysql/000000_init.down.sql new file mode 100644 index 0000000000..1769db40eb --- /dev/null +++ b/migrations/mysql/000000_init.down.sql @@ -0,0 +1,59 @@ +-- Reverse of 000000_init.up.sql: drop all tables in reverse dependency +-- order. Foreign keys are disabled so drop order within the SET block +-- is tolerant of any remaining references. + +SET FOREIGN_KEY_CHECKS = 0; + +DROP TABLE IF EXISTS temporary_documents; +DROP TABLE IF EXISTS tenant_api_keys; +DROP TABLE IF EXISTS resource_access_grants; +DROP TABLE IF EXISTS resource_bindings; +DROP TABLE IF EXISTS resources; +DROP TABLE IF EXISTS storage_backends; +DROP TABLE IF EXISTS message_suggestion_events; +DROP TABLE IF EXISTS message_suggestion_sets; +DROP TABLE IF EXISTS knowledge_processing_spans; +DROP TABLE IF EXISTS system_settings; +DROP TABLE IF EXISTS user_kb_pins; +DROP TABLE IF EXISTS tenant_invitations; +DROP TABLE IF EXISTS user_resource_favorites; +DROP TABLE IF EXISTS audit_logs; +DROP TABLE IF EXISTS tenant_members; +DROP TABLE IF EXISTS task_dead_letters; +DROP TABLE IF EXISTS task_pending_ops; +DROP TABLE IF EXISTS wiki_page_issues; +DROP TABLE IF EXISTS wiki_page_revisions; +DROP TABLE IF EXISTS wiki_pages; +DROP TABLE IF EXISTS wiki_folders; +DROP TABLE IF EXISTS embed_channels; +DROP TABLE IF EXISTS vector_stores; +DROP TABLE IF EXISTS web_search_providers; +DROP TABLE IF EXISTS sync_logs; +DROP TABLE IF EXISTS data_sources; +DROP TABLE IF EXISTS im_channel_sessions; +DROP TABLE IF EXISTS im_channels; +DROP TABLE IF EXISTS tenant_disabled_shared_agents; +DROP TABLE IF EXISTS agent_shares; +DROP TABLE IF EXISTS kb_shares; +DROP TABLE IF EXISTS organization_tenant_members; +DROP TABLE IF EXISTS organization_join_requests; +DROP TABLE IF EXISTS organizations; +DROP TABLE IF EXISTS custom_agents; +DROP TABLE IF EXISTS mcp_oauth_tokens; +DROP TABLE IF EXISTS mcp_oauth_clients; +DROP TABLE IF EXISTS mcp_tool_approvals; +DROP TABLE IF EXISTS mcp_services; +DROP TABLE IF EXISTS knowledge_tag_relations; +DROP TABLE IF EXISTS knowledge_tags; +DROP TABLE IF EXISTS chunk_revisions; +DROP TABLE IF EXISTS chunks; +DROP TABLE IF EXISTS messages; +DROP TABLE IF EXISTS sessions; +DROP TABLE IF EXISTS knowledges; +DROP TABLE IF EXISTS knowledge_bases; +DROP TABLE IF EXISTS models; +DROP TABLE IF EXISTS auth_tokens; +DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS tenants; + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/migrations/mysql/000000_init.up.sql b/migrations/mysql/000000_init.up.sql new file mode 100644 index 0000000000..16386d1501 --- /dev/null +++ b/migrations/mysql/000000_init.up.sql @@ -0,0 +1,1193 @@ +-- MySQL 8.0.16+ baseline schema for WeKnora metadata layer. +-- Squashed from migrations/versioned/*.up.sql (PostgreSQL, 72 incremental +-- migrations). Fresh MySQL deployments only need this head schema. +-- +-- Requires MySQL 8.0.16+ (CHECK constraint enforcement, JSON expression +-- defaults, utf8mb4_0900_ai_ci). +-- +-- Scope: metadata layer only. The embeddings table is intentionally absent - +-- under DB_DRIVER=mysql, vector retrieval is delegated to an external engine +-- via RETRIEVE_DRIVER. + +SET FOREIGN_KEY_CHECKS = 0; +SET time_zone = '+00:00'; + +-- tenants +CREATE TABLE tenants ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description TEXT, + retriever_engines JSON NOT NULL DEFAULT (JSON_ARRAY()), + status VARCHAR(50) DEFAULT 'active', + business VARCHAR(255) NOT NULL, + storage_quota BIGINT NOT NULL DEFAULT 10737418240, + storage_used BIGINT NOT NULL DEFAULT 0, + agent_config JSON DEFAULT NULL, + context_config JSON DEFAULT NULL, + conversation_config JSON DEFAULT NULL, + web_search_config JSON DEFAULT NULL, + parser_engine_config JSON DEFAULT NULL, + storage_engine_config JSON DEFAULT NULL, + chat_history_config JSON DEFAULT NULL, + retrieval_config JSON DEFAULT NULL, + api_principal_config JSON DEFAULT NULL, + credentials JSON DEFAULT NULL, + default_storage_backend_id VARCHAR(36) DEFAULT NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + INDEX idx_tenants_status (status) +) ENGINE=InnoDB AUTO_INCREMENT=10000 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- users +CREATE TABLE users ( + id VARCHAR(36) PRIMARY KEY, + username VARCHAR(100) NOT NULL, + email VARCHAR(255) NOT NULL, + password_hash VARCHAR(255) NOT NULL, + avatar VARCHAR(500), + tenant_id BIGINT UNSIGNED, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + can_access_all_tenants BOOLEAN NOT NULL DEFAULT FALSE, + is_system_admin BOOLEAN NOT NULL DEFAULT FALSE, + preferences JSON NOT NULL DEFAULT (JSON_OBJECT()), + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + CONSTRAINT users_username_key UNIQUE (username), + CONSTRAINT users_email_key UNIQUE (email), + CONSTRAINT fk_users_tenant FOREIGN KEY (tenant_id) REFERENCES tenants(id) ON DELETE SET NULL, + INDEX idx_users_tenant_id (tenant_id), + INDEX idx_users_deleted_at (deleted_at), + INDEX idx_users_is_system_admin (is_system_admin) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- auth_tokens +CREATE TABLE auth_tokens ( + id VARCHAR(36) PRIMARY KEY, + user_id VARCHAR(36) NOT NULL, + token TEXT CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT (''), + token_type VARCHAR(50) NOT NULL, + expires_at DATETIME(6) NOT NULL, + is_revoked BOOLEAN NOT NULL DEFAULT FALSE, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + CONSTRAINT fk_auth_tokens_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + INDEX idx_auth_tokens_user_id (user_id), + INDEX idx_auth_tokens_token (token(255)), + INDEX idx_auth_tokens_token_type (token_type), + INDEX idx_auth_tokens_expires_at (expires_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- models +CREATE TABLE models ( + id VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + name VARCHAR(255) NOT NULL, + display_name VARCHAR(255) NOT NULL DEFAULT '', + type VARCHAR(50) NOT NULL, + source VARCHAR(50) NOT NULL, + description TEXT, + parameters JSON NOT NULL DEFAULT (JSON_OBJECT()), + is_default BOOLEAN NOT NULL DEFAULT FALSE, + is_builtin BOOLEAN NOT NULL DEFAULT FALSE, + managed_by VARCHAR(32) NOT NULL DEFAULT '', + status VARCHAR(50) NOT NULL DEFAULT 'active', + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + INDEX idx_models_type (type), + INDEX idx_models_source (source), + INDEX idx_models_is_builtin (is_builtin), + INDEX idx_models_managed_by_yaml (managed_by) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- knowledge_bases +CREATE TABLE knowledge_bases ( + id VARCHAR(36) PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description TEXT, + tenant_id BIGINT UNSIGNED NOT NULL, + type VARCHAR(32) NOT NULL DEFAULT 'document', + is_temporary BOOLEAN NOT NULL DEFAULT FALSE, + creator_id VARCHAR(36), + chunking_config JSON NOT NULL DEFAULT (JSON_OBJECT()), + image_processing_config JSON NOT NULL DEFAULT (JSON_OBJECT()), + embedding_model_id VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + summary_model_id VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + cos_config JSON NOT NULL DEFAULT (JSON_OBJECT()), + vlm_config JSON NOT NULL DEFAULT (JSON_OBJECT()), + extract_config JSON NULL DEFAULT NULL, + faq_config JSON DEFAULT NULL, + question_generation_config JSON NULL DEFAULT NULL, + storage_provider_config JSON DEFAULT NULL, + vector_store_id VARCHAR(36), + storage_backend_id VARCHAR(36), + asr_config JSON DEFAULT NULL, + wiki_config JSON DEFAULT NULL, + indexing_strategy JSON DEFAULT NULL, + is_pinned BOOLEAN NOT NULL DEFAULT FALSE, + pinned_at DATETIME(6) NULL DEFAULT NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + INDEX idx_knowledge_bases_tenant_id (tenant_id), + INDEX idx_knowledge_bases_tenant_name (tenant_id, name), + INDEX idx_knowledge_bases_tenant_creator (tenant_id, creator_id), + INDEX idx_knowledge_bases_tenant_vector_store (tenant_id, vector_store_id), + INDEX idx_knowledge_bases_storage_backend (tenant_id, storage_backend_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- knowledges +CREATE TABLE knowledges ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + type VARCHAR(50) NOT NULL, + title VARCHAR(255) NOT NULL, + description TEXT, + source VARCHAR(2048) NOT NULL, + parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed', + enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled', + embedding_model_id VARCHAR(64), + file_name VARCHAR(255), + file_type VARCHAR(50), + file_size BIGINT, + file_path TEXT, + file_hash VARCHAR(64), + storage_size BIGINT NOT NULL DEFAULT 0, + metadata JSON, + custom_metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + channel VARCHAR(50) NOT NULL DEFAULT 'web', + summary_status VARCHAR(32) DEFAULT 'none', + last_faq_import_result JSON DEFAULT NULL, + pending_subtasks_count INT NOT NULL DEFAULT 0, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + processed_at DATETIME(6) NULL, + error_message TEXT, + deleted_at DATETIME(6) NULL DEFAULT NULL, + -- Materializes metadata->>'$.external_id' so the datasource sync lookups + -- can be indexed, mirroring the PostgreSQL expression index + -- idx_knowledges_kb_metadata_external_id from migration 000076. + -- + -- LONGTEXT rather than VARCHAR(n): the extracted value has no length bound + -- on PostgreSQL, and a typed generated column rejects the whole INSERT with + -- error 1406 once a value overflows it. utf8mb4_bin matches the PostgreSQL + -- index's text_pattern_ops so LIKE-prefix comparisons order bytewise. + metadata_external_id LONGTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin + GENERATED ALWAYS AS (metadata ->> '$.external_id') VIRTUAL, + INDEX idx_knowledges_tenant_id (tenant_id), + INDEX idx_knowledges_base_created (knowledge_base_id, created_at), + INDEX idx_knowledges_parse_status (parse_status), + INDEX idx_knowledges_enable_status (enable_status), + INDEX idx_knowledges_summary_status (summary_status), + INDEX idx_knowledges_kb_metadata_external_id (knowledge_base_id, metadata_external_id(191)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- sessions +CREATE TABLE sessions ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + user_id VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin, + title VARCHAR(255), + description TEXT, + knowledge_base_id VARCHAR(36), + agent_id VARCHAR(36), + max_rounds INT NOT NULL DEFAULT 5, + enable_rewrite BOOLEAN NOT NULL DEFAULT TRUE, + fallback_strategy VARCHAR(255) NOT NULL DEFAULT 'fixed', + fallback_response TEXT NOT NULL DEFAULT ('很抱歉，我暂时无法回答这个问题。'), + keyword_threshold FLOAT NOT NULL DEFAULT 0.5, + vector_threshold FLOAT NOT NULL DEFAULT 0.5, + rerank_model_id VARCHAR(64), + embedding_top_k INTEGER NOT NULL DEFAULT 10, + rerank_top_k INTEGER NOT NULL DEFAULT 10, + rerank_threshold FLOAT NOT NULL DEFAULT 0.65, + summary_model_id VARCHAR(64), + summary_parameters JSON NOT NULL DEFAULT (JSON_OBJECT()), + agent_config JSON DEFAULT NULL, + context_config JSON DEFAULT NULL, + is_pinned BOOLEAN NOT NULL DEFAULT FALSE, + pinned_at DATETIME(6) NULL DEFAULT NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + INDEX idx_sessions_tenant_updated (tenant_id, updated_at), + INDEX idx_sessions_agent_id (agent_id), + INDEX idx_sessions_tenant_user_pin (tenant_id, user_id, is_pinned, pinned_at, updated_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- messages +CREATE TABLE messages ( + id VARCHAR(36) PRIMARY KEY, + request_id VARCHAR(36) NOT NULL, + session_id VARCHAR(36) NOT NULL, + role VARCHAR(50) NOT NULL, + content MEDIUMTEXT NOT NULL DEFAULT (''), + knowledge_references JSON NOT NULL DEFAULT (JSON_ARRAY()), + agent_steps JSON DEFAULT NULL, + is_completed BOOLEAN NOT NULL DEFAULT FALSE, + is_fallback BOOLEAN DEFAULT FALSE, + mentioned_items JSON DEFAULT (JSON_ARRAY()), + images JSON DEFAULT (JSON_ARRAY()), + attachments JSON DEFAULT (JSON_ARRAY()), + agent_duration_ms BIGINT DEFAULT 0, + rendered_content TEXT NOT NULL DEFAULT (''), + channel VARCHAR(50) NOT NULL DEFAULT '', + agent_id VARCHAR(36) NOT NULL DEFAULT '', + agent_tenant_id BIGINT UNSIGNED NOT NULL DEFAULT 0, + model_id VARCHAR(64) NOT NULL DEFAULT '', + execution_context JSON NOT NULL DEFAULT (JSON_OBJECT()), + knowledge_id VARCHAR(36), + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + INDEX idx_messages_session_created (session_id, created_at), + INDEX idx_messages_agent_id (agent_id), + INDEX idx_messages_knowledge_id (knowledge_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- chunks +CREATE TABLE chunks ( + id VARCHAR(36) PRIMARY KEY, + seq_id BIGINT UNSIGNED AUTO_INCREMENT, + tenant_id BIGINT UNSIGNED NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + knowledge_id VARCHAR(36) NOT NULL, + content MEDIUMTEXT NOT NULL DEFAULT (''), + source_content TEXT NOT NULL DEFAULT (''), + content_revision INT NOT NULL DEFAULT 0, + index_status VARCHAR(16) NOT NULL DEFAULT 'ready', + last_editor_id VARCHAR(64) NOT NULL DEFAULT '', + context_header TEXT NOT NULL DEFAULT (''), + chunk_index INTEGER NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + flags INTEGER NOT NULL DEFAULT 1, + status INT NOT NULL DEFAULT 0, + start_at INTEGER NOT NULL, + end_at INTEGER NOT NULL, + pre_chunk_id VARCHAR(36), + next_chunk_id VARCHAR(36), + chunk_type VARCHAR(20) NOT NULL DEFAULT 'text', + parent_chunk_id VARCHAR(36), + image_info TEXT, + video_info TEXT, + relation_chunks JSON, + indirect_relation_chunks JSON, + metadata JSON, + tag_id VARCHAR(36), + content_hash VARCHAR(64), + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + UNIQUE idx_chunks_seq_id (seq_id), + INDEX idx_chunks_tenant_kg (tenant_id, knowledge_id), + INDEX idx_chunks_parent_id (parent_chunk_id), + INDEX idx_chunks_chunk_type (chunk_type), + INDEX idx_chunks_tag (tag_id), + INDEX idx_chunks_content_hash (content_hash), + INDEX idx_chunks_kb_tenant (knowledge_base_id, tenant_id), + INDEX idx_chunks_knowledge_enabled (knowledge_id, is_enabled, deleted_at) +-- AUTO_INCREMENT matches the start value of the PostgreSQL chunks_seq_id_seq +-- sequence. FAQ import lets callers pin a seq_id below it (see +-- types.FAQImportEntry.ID), so anything below 100000000 is a reserved range +-- that generated values must never enter. +) ENGINE=InnoDB AUTO_INCREMENT=100000000 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- chunk_revisions +CREATE TABLE chunk_revisions ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + knowledge_id VARCHAR(36) NOT NULL, + chunk_id VARCHAR(36) NOT NULL, + revision INT NOT NULL, + content TEXT NOT NULL DEFAULT (''), + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + editor_id VARCHAR(64) NOT NULL DEFAULT '', + edit_source VARCHAR(16) NOT NULL DEFAULT 'user', + edited_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + UNIQUE idx_chunk_revisions_chunk_revision (chunk_id, revision), + INDEX idx_chunk_revisions_tenant_chunk (tenant_id, chunk_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- knowledge_tags +CREATE TABLE knowledge_tags ( + id VARCHAR(36) PRIMARY KEY, + seq_id BIGINT UNSIGNED AUTO_INCREMENT, + tenant_id BIGINT UNSIGNED NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + name VARCHAR(128) NOT NULL, + color VARCHAR(32), + sort_order INTEGER NOT NULL DEFAULT 0, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + UNIQUE idx_knowledge_tags_seq_id (seq_id), + UNIQUE idx_knowledge_tags_kb_name (tenant_id, knowledge_base_id, name), + INDEX idx_knowledge_tags_kb (tenant_id, knowledge_base_id) +-- Matches the start value of the PostgreSQL knowledge_tags_seq_id_seq sequence. +) ENGINE=InnoDB AUTO_INCREMENT=10000000 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- knowledge_tag_relations +CREATE TABLE knowledge_tag_relations ( + knowledge_id VARCHAR(36) NOT NULL, + tag_id VARCHAR(36) NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + PRIMARY KEY (knowledge_id, tag_id), + INDEX idx_ktr_knowledge (knowledge_id), + INDEX idx_ktr_tag (tag_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- mcp_services +CREATE TABLE mcp_services ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT, + enabled BOOLEAN DEFAULT TRUE, + transport_type VARCHAR(50) NOT NULL, + url VARCHAR(512), + headers JSON, + auth_config JSON, + advanced_config JSON, + stdio_config JSON, + env_vars JSON, + is_builtin BOOLEAN NOT NULL DEFAULT FALSE, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + INDEX idx_mcp_services_tenant_id (tenant_id), + INDEX idx_mcp_services_enabled (enabled), + INDEX idx_mcp_services_deleted_at (deleted_at), + INDEX idx_mcp_services_is_builtin (is_builtin) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- mcp_tool_approvals +CREATE TABLE mcp_tool_approvals ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + service_id VARCHAR(36) NOT NULL, + tool_name VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + require_approval BOOLEAN NOT NULL DEFAULT FALSE, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + CONSTRAINT fk_mcp_tool_approvals_service FOREIGN KEY (service_id) REFERENCES mcp_services(id) ON DELETE CASCADE, + UNIQUE idx_mcp_tool_approvals_tenant_svc_tool (tenant_id, service_id, tool_name), + INDEX idx_mcp_tool_approvals_service_id (service_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- mcp_oauth_clients +CREATE TABLE mcp_oauth_clients ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + service_id VARCHAR(36) NOT NULL, + client_id VARCHAR(512) NOT NULL, + client_secret TEXT, + redirect_uri VARCHAR(1024), + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + CONSTRAINT fk_mcp_oauth_clients_service FOREIGN KEY (service_id) REFERENCES mcp_services(id) ON DELETE CASCADE, + UNIQUE idx_mcp_oauth_clients_tenant_svc (tenant_id, service_id), + INDEX idx_mcp_oauth_clients_service_id (service_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- mcp_oauth_tokens +CREATE TABLE mcp_oauth_tokens ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + user_id VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + service_id VARCHAR(36) NOT NULL, + principal_type VARCHAR(32) NOT NULL, + principal_id VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + access_token TEXT, + refresh_token TEXT, + token_type VARCHAR(32), + expires_at DATETIME(6) NULL, + refresh_lease_id VARCHAR(36) CHARACTER SET ascii COLLATE ascii_bin NULL, + refresh_lease_until DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + CONSTRAINT fk_mcp_oauth_tokens_service FOREIGN KEY (service_id) REFERENCES mcp_services(id) ON DELETE CASCADE, + UNIQUE idx_mcp_oauth_tokens_tenant_principal_svc (tenant_id, principal_type, principal_id, service_id), + INDEX idx_mcp_oauth_tokens_service_id (service_id), + INDEX idx_mcp_oauth_tokens_user_id (user_id), + INDEX idx_mcp_oauth_tokens_principal (principal_type, principal_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- custom_agents +CREATE TABLE custom_agents ( + id VARCHAR(36) NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT, + avatar VARCHAR(64), + is_builtin BOOLEAN NOT NULL DEFAULT FALSE, + tenant_id BIGINT UNSIGNED NOT NULL, + created_by VARCHAR(36), + runnable_by_viewer BOOLEAN NOT NULL DEFAULT TRUE, + config JSON NOT NULL DEFAULT (JSON_OBJECT()), + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + PRIMARY KEY (id, tenant_id), + INDEX idx_custom_agents_tenant_id (tenant_id), + INDEX idx_custom_agents_is_builtin (is_builtin), + INDEX idx_custom_agents_deleted_at (deleted_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- organizations +CREATE TABLE organizations ( + id VARCHAR(36) PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description TEXT, + owner_id VARCHAR(36) NOT NULL, + owner_tenant_id BIGINT UNSIGNED NOT NULL, + invite_code VARCHAR(32), + require_approval BOOLEAN DEFAULT FALSE, + invite_code_expires_at DATETIME(6) NULL, + invite_code_validity_days SMALLINT NOT NULL DEFAULT 7, + avatar VARCHAR(512) DEFAULT '', + searchable BOOLEAN NOT NULL DEFAULT FALSE, + member_limit INTEGER NOT NULL DEFAULT 50, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + live_invite_code VARCHAR(32) GENERATED ALWAYS AS (CASE WHEN deleted_at IS NULL THEN invite_code ELSE NULL END) VIRTUAL, + UNIQUE idx_organizations_live_invite_code (live_invite_code), + INDEX idx_organizations_owner_id (owner_id), + INDEX idx_organizations_owner_tenant (owner_tenant_id), + INDEX idx_organizations_deleted_at (deleted_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- organization_join_requests +CREATE TABLE organization_join_requests ( + id VARCHAR(36) PRIMARY KEY, + organization_id VARCHAR(36) NOT NULL, + user_id VARCHAR(36) NOT NULL, + tenant_id BIGINT UNSIGNED NOT NULL, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + requested_role VARCHAR(32) NOT NULL DEFAULT 'viewer', + request_type VARCHAR(32) NOT NULL DEFAULT 'join', + prev_role VARCHAR(32), + message TEXT, + reviewed_by VARCHAR(36), + reviewed_at DATETIME(6) NULL, + review_message TEXT, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + CONSTRAINT fk_org_join_requests_org FOREIGN KEY (organization_id) REFERENCES organizations(id) ON DELETE CASCADE, + pending_marker CHAR(1) GENERATED ALWAYS AS (CASE WHEN status = 'pending' THEN '1' ELSE NULL END) VIRTUAL, + UNIQUE uq_org_join_requests_pending_live (organization_id, tenant_id, request_type, pending_marker), + INDEX idx_org_join_requests_org_id (organization_id), + INDEX idx_org_join_requests_user_id (user_id), + INDEX idx_org_join_requests_status (status), + INDEX idx_org_join_requests_type (request_type) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- organization_tenant_members +CREATE TABLE organization_tenant_members ( + id VARCHAR(36) PRIMARY KEY, + organization_id VARCHAR(36) NOT NULL, + tenant_id BIGINT UNSIGNED NOT NULL, + role VARCHAR(32) NOT NULL DEFAULT 'viewer', + representative_user_id VARCHAR(36) NOT NULL DEFAULT '', + joined_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + CONSTRAINT fk_org_tenant_members_org FOREIGN KEY (organization_id) REFERENCES organizations(id) ON DELETE CASCADE, + UNIQUE idx_org_tenant_members_unique (organization_id, tenant_id), + INDEX idx_org_tenant_members_by_tenant (tenant_id), + INDEX idx_org_tenant_members_role (organization_id, role) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- kb_shares +CREATE TABLE kb_shares ( + id VARCHAR(36) PRIMARY KEY, + knowledge_base_id VARCHAR(36) NOT NULL, + organization_id VARCHAR(36) NOT NULL, + shared_by_user_id VARCHAR(36) NOT NULL, + source_tenant_id BIGINT UNSIGNED NOT NULL, + permission VARCHAR(32) NOT NULL DEFAULT 'viewer', + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + CONSTRAINT fk_kb_shares_kb FOREIGN KEY (knowledge_base_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE, + CONSTRAINT fk_kb_shares_org FOREIGN KEY (organization_id) REFERENCES organizations(id) ON DELETE CASCADE, + INDEX idx_kb_shares_kb_id (knowledge_base_id), + INDEX idx_kb_shares_org_id (organization_id), + INDEX idx_kb_shares_source_tenant (source_tenant_id), + INDEX idx_kb_shares_deleted_at (deleted_at), + live_marker CHAR(1) GENERATED ALWAYS AS (CASE WHEN deleted_at IS NULL THEN '1' ELSE NULL END) VIRTUAL, + UNIQUE idx_kb_shares_live (knowledge_base_id, organization_id, live_marker) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- agent_shares +CREATE TABLE agent_shares ( + id VARCHAR(36) PRIMARY KEY, + agent_id VARCHAR(36) NOT NULL, + organization_id VARCHAR(36) NOT NULL, + shared_by_user_id VARCHAR(36) NOT NULL, + source_tenant_id BIGINT UNSIGNED NOT NULL, + permission VARCHAR(32) NOT NULL DEFAULT 'viewer', + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + CONSTRAINT fk_agent_shares_org FOREIGN KEY (organization_id) REFERENCES organizations(id) ON DELETE CASCADE, + CONSTRAINT fk_agent_shares_agent FOREIGN KEY (agent_id, source_tenant_id) REFERENCES custom_agents(id, tenant_id) ON DELETE CASCADE, + INDEX idx_agent_shares_agent_id (agent_id), + INDEX idx_agent_shares_org_id (organization_id), + INDEX idx_agent_shares_source_tenant (source_tenant_id), + INDEX idx_agent_shares_deleted_at (deleted_at), + live_marker CHAR(1) GENERATED ALWAYS AS (CASE WHEN deleted_at IS NULL THEN '1' ELSE NULL END) VIRTUAL, + UNIQUE idx_agent_shares_live (agent_id, source_tenant_id, organization_id, live_marker) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- tenant_disabled_shared_agents +CREATE TABLE tenant_disabled_shared_agents ( + tenant_id BIGINT UNSIGNED NOT NULL, + agent_id VARCHAR(36) NOT NULL, + source_tenant_id BIGINT UNSIGNED NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + PRIMARY KEY (tenant_id, agent_id, source_tenant_id), + INDEX idx_tenant_disabled_shared_agents_tenant_id (tenant_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- im_channels +CREATE TABLE im_channels ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + agent_id VARCHAR(36) NOT NULL, + platform VARCHAR(20) NOT NULL, + name VARCHAR(255) NOT NULL DEFAULT '', + enabled BOOLEAN NOT NULL DEFAULT TRUE, + mode VARCHAR(20) NOT NULL DEFAULT 'websocket', + output_mode VARCHAR(20) NOT NULL DEFAULT 'stream', + credentials JSON NOT NULL DEFAULT (JSON_OBJECT()), + knowledge_base_id VARCHAR(36) DEFAULT '', + bot_identity VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT '', + session_mode VARCHAR(20) NOT NULL DEFAULT 'user', + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + CONSTRAINT chk_im_channels_session_mode CHECK (session_mode IN ('user', 'thread')), + INDEX idx_im_channels_tenant (tenant_id), + INDEX idx_im_channels_agent (agent_id), + INDEX idx_im_channels_deleted (deleted_at), + live_bot_identity VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin GENERATED ALWAYS AS (CASE WHEN deleted_at IS NULL AND bot_identity <> '' THEN bot_identity ELSE NULL END) VIRTUAL, + UNIQUE idx_im_channels_live_bot_identity (live_bot_identity) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- im_channel_sessions +CREATE TABLE im_channel_sessions ( + id VARCHAR(36) PRIMARY KEY, + platform VARCHAR(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + user_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, + chat_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT '', + session_id VARCHAR(36) NOT NULL, + tenant_id BIGINT UNSIGNED NOT NULL, + agent_id VARCHAR(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin DEFAULT '', + im_channel_id VARCHAR(36) DEFAULT '', + status VARCHAR(20) NOT NULL DEFAULT 'active', + thread_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT '', + metadata JSON DEFAULT (JSON_OBJECT()), + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + CONSTRAINT fk_im_channel_sessions_session FOREIGN KEY (session_id) REFERENCES sessions(id) ON DELETE CASCADE, + INDEX idx_im_channel_tenant (tenant_id), + INDEX idx_im_channel_session (session_id), + INDEX idx_im_channel_deleted (deleted_at), + -- PostgreSQL indexes this partially (WHERE im_channel_id <> ''). MySQL has + -- no partial indexes; a full index covers the same lookups and only costs + -- the extra entries for rows whose channel is unset. + INDEX idx_im_channel_sessions_channel (im_channel_id), + live_marker CHAR(1) GENERATED ALWAYS AS (CASE WHEN deleted_at IS NULL THEN '1' ELSE NULL END) VIRTUAL, + live_thread_marker CHAR(1) GENERATED ALWAYS AS (CASE WHEN deleted_at IS NULL AND thread_id <> '' THEN '1' ELSE NULL END) VIRTUAL, + UNIQUE idx_im_channel_sessions_live_channel (platform, user_id, chat_id, tenant_id, agent_id, live_marker), + UNIQUE idx_im_channel_sessions_live_thread (platform, chat_id, thread_id, tenant_id, agent_id, live_thread_marker) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- data_sources +CREATE TABLE data_sources ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + name VARCHAR(255) NOT NULL, + type VARCHAR(50) NOT NULL, + config JSON, + sync_schedule VARCHAR(100), + sync_mode VARCHAR(20) DEFAULT 'incremental', + status VARCHAR(32) DEFAULT 'active', + conflict_strategy VARCHAR(32) DEFAULT 'overwrite', + sync_deletions BOOLEAN DEFAULT TRUE, + last_sync_at DATETIME(6) NULL, + last_sync_cursor JSON, + last_sync_result JSON, + error_message TEXT, + sync_log_retention_days INT DEFAULT 30, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + INDEX idx_data_sources_tenant_id (tenant_id), + INDEX idx_data_sources_knowledge_base_id (knowledge_base_id), + INDEX idx_data_sources_type (type), + INDEX idx_data_sources_status (status), + INDEX idx_data_sources_deleted_at (deleted_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- sync_logs +CREATE TABLE sync_logs ( + id VARCHAR(36) PRIMARY KEY, + data_source_id VARCHAR(36) NOT NULL, + tenant_id BIGINT UNSIGNED NOT NULL, + status VARCHAR(32) NOT NULL, + started_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + finished_at DATETIME(6) NULL, + items_total INT DEFAULT 0, + items_created INT DEFAULT 0, + items_updated INT DEFAULT 0, + items_deleted INT DEFAULT 0, + items_skipped INT DEFAULT 0, + items_failed INT DEFAULT 0, + error_message TEXT, + result JSON, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + CONSTRAINT fk_sync_logs_data_source FOREIGN KEY (data_source_id) REFERENCES data_sources(id) ON DELETE CASCADE, + INDEX idx_sync_logs_ds_started (data_source_id, started_at), + INDEX idx_sync_logs_tenant_id (tenant_id), + INDEX idx_sync_logs_status (status) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- web_search_providers +CREATE TABLE web_search_providers ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + name VARCHAR(255) NOT NULL, + provider VARCHAR(50) NOT NULL, + description TEXT, + parameters JSON, + is_default BOOLEAN DEFAULT FALSE, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + INDEX idx_web_search_providers_tenant_id (tenant_id), + INDEX idx_web_search_providers_provider (provider), + INDEX idx_web_search_providers_deleted_at (deleted_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- vector_stores +CREATE TABLE vector_stores ( + id VARCHAR(36) NOT NULL PRIMARY KEY, + name VARCHAR(255) NOT NULL, + engine_type VARCHAR(50) NOT NULL, + connection_config JSON NOT NULL DEFAULT (JSON_OBJECT()), + index_config JSON NOT NULL DEFAULT (JSON_OBJECT()), + tenant_id BIGINT UNSIGNED NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + live_marker CHAR(1) GENERATED ALWAYS AS (CASE WHEN deleted_at IS NULL THEN '1' ELSE NULL END) VIRTUAL, + UNIQUE idx_vector_stores_name_tenant_live (name, tenant_id, live_marker), + INDEX idx_vector_stores_tenant_id (tenant_id), + INDEX idx_vector_stores_engine_type (engine_type), + INDEX idx_vector_stores_deleted_at (deleted_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- embed_channels +CREATE TABLE embed_channels ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + agent_id VARCHAR(36) NOT NULL DEFAULT 'builtin-quick-answer', + name VARCHAR(255) NOT NULL DEFAULT '', + enabled BOOLEAN NOT NULL DEFAULT TRUE, + publish_token VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', + allowed_origins JSON NOT NULL DEFAULT (JSON_ARRAY()), + welcome_message TEXT NOT NULL DEFAULT (''), + rate_limit_per_minute INTEGER NOT NULL DEFAULT 30, + rate_limit_per_day INTEGER NOT NULL DEFAULT 10000, + primary_color VARCHAR(32) NOT NULL DEFAULT '', + page_title VARCHAR(255) NOT NULL DEFAULT '', + header_title_mode VARCHAR(32) NOT NULL DEFAULT 'channel', + show_suggested_questions BOOLEAN NOT NULL DEFAULT TRUE, + widget_position VARCHAR(32) NOT NULL DEFAULT 'bottom-right', + allow_web_search BOOLEAN NOT NULL DEFAULT FALSE, + allow_memory BOOLEAN NOT NULL DEFAULT FALSE, + allow_file_upload BOOLEAN NOT NULL DEFAULT FALSE, + default_locale VARCHAR(16) NOT NULL DEFAULT '', + webhook_url VARCHAR(512) NOT NULL DEFAULT '', + webhook_secret VARCHAR(128) NOT NULL DEFAULT '', + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + INDEX idx_embed_channels_tenant (tenant_id), + INDEX idx_embed_channels_agent (agent_id), + live_publish_token VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin GENERATED ALWAYS AS (CASE WHEN deleted_at IS NULL AND publish_token <> '' THEN publish_token ELSE NULL END) VIRTUAL, + UNIQUE idx_embed_channels_live_publish_token (live_publish_token), + INDEX idx_embed_channels_deleted (deleted_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- wiki_folders +CREATE TABLE wiki_folders ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL DEFAULT 0, + knowledge_base_id VARCHAR(36) NOT NULL, + parent_id VARCHAR(36) NOT NULL DEFAULT '', + name VARCHAR(255) NOT NULL, + path VARCHAR(1024) NOT NULL DEFAULT '', + depth INT NOT NULL DEFAULT 0, + sort_order INT NOT NULL DEFAULT 0, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + live_marker CHAR(1) GENERATED ALWAYS AS (CASE WHEN deleted_at IS NULL THEN '1' ELSE NULL END) VIRTUAL, + UNIQUE idx_wiki_folders_parent_name_live (knowledge_base_id, parent_id, name, live_marker), + INDEX idx_wiki_folders_parent (knowledge_base_id, parent_id), + INDEX idx_wiki_folders_deleted_at (deleted_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- wiki_pages +CREATE TABLE wiki_pages ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + slug VARCHAR(255) NOT NULL, + title VARCHAR(512) NOT NULL DEFAULT '', + page_type VARCHAR(32) NOT NULL DEFAULT 'summary', + status VARCHAR(32) NOT NULL DEFAULT 'published', + content TEXT NOT NULL DEFAULT (''), + summary TEXT NOT NULL DEFAULT (''), + parent_slug VARCHAR(255) NOT NULL DEFAULT '', + folder_id VARCHAR(36) NOT NULL DEFAULT '', + category_path JSON, + wiki_path VARCHAR(1024) NOT NULL DEFAULT '', + depth INT NOT NULL DEFAULT 0, + sort_order INT NOT NULL DEFAULT 0, + source_refs JSON, + chunk_refs JSON, + in_links JSON, + out_links JSON, + page_metadata JSON, + aliases JSON, + version INT NOT NULL DEFAULT 1, + last_edit_source VARCHAR(16) NOT NULL DEFAULT '', + last_editor_id VARCHAR(64) NOT NULL DEFAULT '', + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + live_marker CHAR(1) GENERATED ALWAYS AS (CASE WHEN deleted_at IS NULL THEN '1' ELSE NULL END) VIRTUAL, + UNIQUE idx_wiki_pages_kb_slug_live (knowledge_base_id, slug, live_marker), + INDEX idx_wiki_pages_kb_id (knowledge_base_id), + INDEX idx_wiki_pages_page_type (knowledge_base_id, page_type), + INDEX idx_wiki_pages_parent_slug (knowledge_base_id, parent_slug), + -- MySQL InnoDB limits index keys to 3072 bytes. wiki_path (VARCHAR(1024)) + -- and title (VARCHAR(512)) under utf8mb4 would exceed that, so both get + -- indexed with a prefix. The tree query uses equality on knowledge_base_id + -- + page_type + wiki_path prefix, then ORDER BY sort_order (INT, full). + INDEX idx_wiki_pages_tree (knowledge_base_id, page_type, wiki_path(480), sort_order, title(200)), + INDEX idx_wiki_pages_folder (knowledge_base_id, folder_id), + INDEX idx_wiki_pages_tenant_id (tenant_id), + INDEX idx_wiki_pages_deleted_at (deleted_at) + -- NOTE(perf): PG had 4 GIN indexes on this table (to_tsvector full- + -- text, source_refs jsonb_path_ops containment, source_refs::text + -- fulltext, lower(title) gin_trgm_ops similarity). MySQL has no direct + -- equivalents, so wiki search/ListBySourceRef/FindSimilarPages use + -- multi-column LIKE + JSON_CONTAINS instead. + -- + -- LIKE substring matching is not semantically equivalent to PostgreSQL + -- full-text search, and JSON_CONTAINS has different indexing options from + -- jsonb_path_ops. Any future FULLTEXT or multi-valued index change needs + -- explicit matching and performance validation for the supported workload. +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- wiki_page_revisions +CREATE TABLE wiki_page_revisions ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + page_id VARCHAR(36) NOT NULL, + slug VARCHAR(255) NOT NULL, + version INT NOT NULL, + title VARCHAR(512) NOT NULL DEFAULT '', + page_type VARCHAR(32) NOT NULL DEFAULT 'summary', + status VARCHAR(32) NOT NULL DEFAULT 'published', + content TEXT NOT NULL DEFAULT (''), + summary TEXT NOT NULL DEFAULT (''), + aliases JSON NOT NULL DEFAULT (JSON_ARRAY()), + edit_source VARCHAR(16) NOT NULL DEFAULT '', + editor_id VARCHAR(64) NOT NULL DEFAULT '', + edited_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + UNIQUE idx_wiki_page_revisions_page_version (page_id, version), + INDEX idx_wiki_page_revisions_kb_slug (knowledge_base_id, slug) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- wiki_page_issues +CREATE TABLE wiki_page_issues ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + slug VARCHAR(255) NOT NULL, + issue_type VARCHAR(50) NOT NULL, + description TEXT NOT NULL DEFAULT (''), + suspected_knowledge_ids JSON, + status VARCHAR(20) NOT NULL DEFAULT 'pending', + reported_by VARCHAR(100) NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + INDEX idx_wiki_page_issues_tenant_id (tenant_id), + INDEX idx_wiki_page_issues_kb_created (knowledge_base_id, created_at), + INDEX idx_wiki_page_issues_slug (slug), + INDEX idx_wiki_page_issues_status (status) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- task_pending_ops +CREATE TABLE task_pending_ops ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + task_type VARCHAR(64) NOT NULL, + scope VARCHAR(32) NOT NULL, + scope_id VARCHAR(64) NOT NULL, + op VARCHAR(32) NOT NULL, + dedup_key VARCHAR(128) NOT NULL DEFAULT '', + payload JSON NOT NULL DEFAULT (JSON_OBJECT()), + fail_count INT NOT NULL DEFAULT 0, + enqueued_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + claimed_at DATETIME(6) NULL DEFAULT NULL, + INDEX idx_task_pending_ops_scope (task_type, scope, scope_id, id), + INDEX idx_task_pending_ops_tenant (tenant_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- task_dead_letters +CREATE TABLE task_dead_letters ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + task_type VARCHAR(64) NOT NULL, + scope VARCHAR(32) NOT NULL, + scope_id VARCHAR(64) NOT NULL, + related_id VARCHAR(64) NOT NULL DEFAULT '', + payload JSON NOT NULL DEFAULT (JSON_OBJECT()), + last_error TEXT NOT NULL DEFAULT (''), + fail_count INT NOT NULL, + failed_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + INDEX idx_task_dead_letters_scope (scope, scope_id, failed_at DESC), + INDEX idx_task_dead_letters_tenant (tenant_id, failed_at DESC), + INDEX idx_task_dead_letters_task_type (task_type, failed_at DESC) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- tenant_members +CREATE TABLE tenant_members ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + user_id VARCHAR(36) NOT NULL, + tenant_id BIGINT UNSIGNED NOT NULL, + role VARCHAR(20) NOT NULL DEFAULT 'contributor', + status VARCHAR(20) NOT NULL DEFAULT 'active', + invited_by VARCHAR(36), + joined_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + live_marker CHAR(1) GENERATED ALWAYS AS (CASE WHEN deleted_at IS NULL THEN '1' ELSE NULL END) VIRTUAL, + UNIQUE idx_tenant_members_user_tenant_live (user_id, tenant_id, live_marker), + INDEX idx_tenant_members_tenant_role (tenant_id, role), + INDEX idx_tenant_members_user (user_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- audit_logs +CREATE TABLE audit_logs ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + actor_user_id VARCHAR(36) NOT NULL DEFAULT '', + actor_role VARCHAR(32) NOT NULL DEFAULT '', + action VARCHAR(64) NOT NULL, + scope_type VARCHAR(32) NOT NULL DEFAULT '', + scope_id VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL DEFAULT '', + target_type VARCHAR(32) NOT NULL DEFAULT '', + target_id VARCHAR(64) NOT NULL DEFAULT '', + target_user_id VARCHAR(36) NOT NULL DEFAULT '', + request_path VARCHAR(512) NOT NULL DEFAULT '', + request_method VARCHAR(16) NOT NULL DEFAULT '', + outcome VARCHAR(16) NOT NULL DEFAULT 'success', + details JSON NOT NULL DEFAULT (JSON_OBJECT()), + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + INDEX idx_audit_logs_tenant_id_desc (tenant_id, id DESC), + INDEX idx_audit_logs_actor (actor_user_id), + INDEX idx_audit_logs_tenant_action (tenant_id, action), + INDEX idx_audit_logs_tenant_scope_desc (tenant_id, scope_type, scope_id, id DESC), + INDEX idx_audit_logs_created_at (created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- user_resource_favorites +CREATE TABLE user_resource_favorites ( + user_id VARCHAR(36) NOT NULL, + tenant_id BIGINT UNSIGNED NOT NULL, + resource_type VARCHAR(16) NOT NULL, + resource_id VARCHAR(64) NOT NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + PRIMARY KEY (user_id, tenant_id, resource_type, resource_id), + INDEX idx_user_resource_favorites_user_tenant_type_created_at (user_id, tenant_id, resource_type, created_at DESC), + INDEX idx_user_resource_favorites_tenant_id (tenant_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- tenant_invitations +CREATE TABLE tenant_invitations ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + invitee_user_id VARCHAR(36) NOT NULL DEFAULT '', + invited_by VARCHAR(36), + role VARCHAR(20) NOT NULL, + status VARCHAR(20) NOT NULL DEFAULT 'pending', + message VARCHAR(500), + token VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', + accepted_count INTEGER NOT NULL DEFAULT 0, + expires_at DATETIME(6) NOT NULL, + responded_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + pending_invitee VARCHAR(36) GENERATED ALWAYS AS (CASE WHEN status = 'pending' AND deleted_at IS NULL AND invitee_user_id <> '' THEN invitee_user_id ELSE NULL END) VIRTUAL, + UNIQUE idx_tenant_invitations_live_pending (tenant_id, pending_invitee), + live_token VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin GENERATED ALWAYS AS (CASE WHEN deleted_at IS NULL AND token <> '' THEN token ELSE NULL END) VIRTUAL, + UNIQUE idx_tenant_invitations_live_token (live_token), + INDEX idx_tenant_invitations_tenant (tenant_id), + INDEX idx_tenant_invitations_invitee (invitee_user_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- user_kb_pins +CREATE TABLE user_kb_pins ( + tenant_id BIGINT UNSIGNED NOT NULL, + user_id VARCHAR(36) NOT NULL, + kb_id VARCHAR(36) NOT NULL, + pinned_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + PRIMARY KEY (tenant_id, user_id, kb_id), + INDEX idx_user_kb_pins_user_tenant_pinned_at (tenant_id, user_id, pinned_at DESC) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- system_settings +CREATE TABLE system_settings ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + `key` VARCHAR(128) NOT NULL, + value JSON NOT NULL DEFAULT (JSON_OBJECT()), + value_type VARCHAR(16) NOT NULL, + category VARCHAR(32) NOT NULL, + description TEXT NOT NULL DEFAULT (''), + is_secret BOOLEAN NOT NULL DEFAULT FALSE, + requires_restart BOOLEAN NOT NULL DEFAULT FALSE, + last_modified_by VARCHAR(36) NOT NULL DEFAULT '', + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + UNIQUE idx_system_settings_key (`key`), + INDEX idx_system_settings_category (category) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- knowledge_processing_spans +CREATE TABLE knowledge_processing_spans ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + knowledge_id VARCHAR(64) NOT NULL, + attempt INT NOT NULL DEFAULT 1, + span_id VARCHAR(64) NOT NULL, + parent_span_id VARCHAR(64), + name VARCHAR(255) NOT NULL, + kind VARCHAR(16) NOT NULL, + status VARCHAR(16) NOT NULL, + input JSON, + output JSON, + metadata JSON, + error_code VARCHAR(64), + error_message TEXT, + error_detail TEXT, + started_at DATETIME(6) NULL, + finished_at DATETIME(6) NULL, + duration_ms BIGINT, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + CONSTRAINT uq_kpspan_attempt_span UNIQUE (knowledge_id, attempt, span_id), + INDEX idx_kpspan_knowledge_attempt (knowledge_id, attempt), + INDEX idx_kpspan_status_started (status, started_at), + INDEX idx_kpspan_parent (parent_span_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- message_suggestion_sets +CREATE TABLE message_suggestion_sets ( + id VARCHAR(36) PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + session_id VARCHAR(36) NOT NULL, + assistant_message_id VARCHAR(36) NOT NULL, + agent_id VARCHAR(36) NOT NULL DEFAULT '', + agent_tenant_id BIGINT UNSIGNED NOT NULL DEFAULT 0, + placement VARCHAR(32) NOT NULL, + config_hash VARCHAR(64) NOT NULL, + locale VARCHAR(16) NOT NULL DEFAULT '', + status VARCHAR(16) NOT NULL, + allow_regenerate BOOLEAN NOT NULL DEFAULT FALSE, + suppression_reason VARCHAR(64) NOT NULL DEFAULT '', + questions JSON NOT NULL DEFAULT (JSON_ARRAY()), + model_id VARCHAR(64) NOT NULL DEFAULT '', + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + latency_ms BIGINT NOT NULL DEFAULT 0, + error_code VARCHAR(64) NOT NULL DEFAULT '', + lease_until DATETIME(6) NULL, + generated_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + CONSTRAINT fk_message_suggestion_sets_tenant FOREIGN KEY (tenant_id) REFERENCES tenants(id) ON DELETE CASCADE, + UNIQUE idx_message_suggestion_sets_cache_key (tenant_id, assistant_message_id, placement, config_hash, locale), + INDEX idx_message_suggestion_sets_session (tenant_id, session_id, created_at), + INDEX idx_message_suggestion_sets_status (status, lease_until) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- message_suggestion_events +CREATE TABLE message_suggestion_events ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + session_id VARCHAR(36) NOT NULL, + suggestion_set_id VARCHAR(36) NOT NULL, + question_id VARCHAR(64) NOT NULL DEFAULT '', + event_type VARCHAR(32) NOT NULL, + actor_id VARCHAR(512) NOT NULL DEFAULT '', + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + CONSTRAINT fk_message_suggestion_events_tenant FOREIGN KEY (tenant_id) REFERENCES tenants(id) ON DELETE CASCADE, + CONSTRAINT fk_message_suggestion_events_set FOREIGN KEY (suggestion_set_id) REFERENCES message_suggestion_sets(id) ON DELETE CASCADE, + INDEX idx_message_suggestion_events_set (suggestion_set_id, created_at), + INDEX idx_message_suggestion_events_session (tenant_id, session_id, created_at), + INDEX idx_message_suggestion_events_type (event_type, created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- storage_backends +CREATE TABLE storage_backends ( + id VARCHAR(36) NOT NULL PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + name VARCHAR(255) NOT NULL, + provider VARCHAR(32) NOT NULL, + config JSON NOT NULL DEFAULT (JSON_OBJECT()), + source VARCHAR(16) NOT NULL DEFAULT 'user', + status VARCHAR(16) NOT NULL DEFAULT 'active', + legacy_alias BOOLEAN NOT NULL DEFAULT FALSE, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + live_marker CHAR(1) GENERATED ALWAYS AS (CASE WHEN deleted_at IS NULL THEN '1' ELSE NULL END) VIRTUAL, + UNIQUE idx_storage_backends_name_live (tenant_id, name, live_marker), + live_legacy_marker CHAR(1) GENERATED ALWAYS AS (CASE WHEN deleted_at IS NULL AND legacy_alias = TRUE THEN '1' ELSE NULL END) VIRTUAL, + UNIQUE idx_storage_backends_legacy_live (tenant_id, provider, live_legacy_marker), + INDEX idx_storage_backends_tenant (tenant_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- resources +CREATE TABLE resources ( + id VARCHAR(36) NOT NULL PRIMARY KEY, + handle VARCHAR(22) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + tenant_id BIGINT UNSIGNED NOT NULL, + storage_backend_id VARCHAR(36), + provider VARCHAR(32) NOT NULL, + physical_path TEXT NOT NULL DEFAULT (''), + location_hash VARCHAR(64) NOT NULL, + kind VARCHAR(32) NOT NULL DEFAULT 'file', + mime_type VARCHAR(255) NOT NULL DEFAULT '', + original_name VARCHAR(1024) NOT NULL DEFAULT '', + size BIGINT NOT NULL DEFAULT 0, + content_hash VARCHAR(64) NOT NULL DEFAULT '', + lifecycle VARCHAR(16) NOT NULL DEFAULT 'persistent', + expires_at DATETIME(6) NULL, + state VARCHAR(16) NOT NULL DEFAULT 'active', + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + UNIQUE idx_resources_handle (handle), + live_marker CHAR(1) GENERATED ALWAYS AS (CASE WHEN deleted_at IS NULL THEN '1' ELSE NULL END) VIRTUAL, + UNIQUE idx_resources_location_live (tenant_id, location_hash, live_marker), + INDEX idx_resources_tenant (tenant_id), + INDEX idx_resources_backend (storage_backend_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- resource_bindings +CREATE TABLE resource_bindings ( + id VARCHAR(36) NOT NULL PRIMARY KEY, + resource_id VARCHAR(36) NOT NULL, + tenant_id BIGINT UNSIGNED NOT NULL, + owner_type VARCHAR(32) NOT NULL, + owner_id VARCHAR(64) NOT NULL, + relation VARCHAR(32) NOT NULL DEFAULT 'attachment', + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + CONSTRAINT fk_resource_bindings_resource FOREIGN KEY (resource_id) REFERENCES resources(id) ON DELETE CASCADE, + UNIQUE idx_resource_bindings_unique (resource_id, owner_type, owner_id, relation), + INDEX idx_resource_bindings_owner (tenant_id, owner_type, owner_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- resource_access_grants +CREATE TABLE resource_access_grants ( + id VARCHAR(36) NOT NULL PRIMARY KEY, + token_hash VARCHAR(64) NOT NULL, + resource_id VARCHAR(36) NOT NULL, + access_scope VARCHAR(16) NOT NULL DEFAULT 'read', + expires_at DATETIME(6) NOT NULL, + revoked_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + CONSTRAINT fk_resource_access_grants_resource FOREIGN KEY (resource_id) REFERENCES resources(id) ON DELETE CASCADE, + UNIQUE idx_resource_access_grants_token_hash (token_hash), + INDEX idx_resource_access_grants_resource (resource_id), + INDEX idx_resource_access_grants_expires (expires_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- tenant_api_keys +CREATE TABLE tenant_api_keys ( + id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, + tenant_id BIGINT UNSIGNED NULL, + name VARCHAR(128) NOT NULL, + key_hash VARCHAR(64) NOT NULL, + api_key TEXT NOT NULL DEFAULT (''), + scope_type VARCHAR(16) NOT NULL DEFAULT 'tenant', + full_access BOOLEAN NOT NULL DEFAULT FALSE, + knowledge_base_ids JSON NOT NULL DEFAULT (JSON_ARRAY()), + capabilities JSON NOT NULL DEFAULT (JSON_ARRAY()), + last_used_at DATETIME(6) NULL, + expires_at DATETIME(6) NULL, + revoked_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + CONSTRAINT fk_tenant_api_keys_tenant FOREIGN KEY (tenant_id) REFERENCES tenants(id) ON DELETE CASCADE, + CONSTRAINT chk_tenant_api_keys_scope CHECK ( + (scope_type = 'tenant' AND tenant_id IS NOT NULL) + OR (scope_type = 'platform' AND tenant_id IS NULL AND full_access = FALSE) + ), + UNIQUE idx_tenant_api_keys_key_hash (key_hash), + INDEX idx_tenant_api_keys_tenant (tenant_id), + INDEX idx_tenant_api_keys_scope_type (scope_type), + INDEX idx_tenant_api_keys_revoked_at (revoked_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- temporary_documents +CREATE TABLE temporary_documents ( + id VARCHAR(36) NOT NULL PRIMARY KEY, + tenant_id BIGINT UNSIGNED NOT NULL, + session_id VARCHAR(36) NOT NULL, + resource_ref TEXT NOT NULL DEFAULT (''), + file_name VARCHAR(1024) NOT NULL, + file_type VARCHAR(32) NOT NULL, + mime_type VARCHAR(255) NOT NULL DEFAULT '', + file_size BIGINT NOT NULL, + status VARCHAR(16) NOT NULL DEFAULT 'uploaded', + content TEXT NOT NULL DEFAULT (''), + chunks JSON NOT NULL DEFAULT (JSON_ARRAY()), + image_refs JSON NOT NULL DEFAULT (JSON_ARRAY()), + metadata JSON NOT NULL DEFAULT (JSON_OBJECT()), + processing_options JSON NOT NULL DEFAULT (JSON_OBJECT()), + token_count INTEGER NOT NULL DEFAULT 0, + chunk_count INTEGER NOT NULL DEFAULT 0, + error_message TEXT NOT NULL DEFAULT (''), + expires_at DATETIME(6) NOT NULL, + started_at DATETIME(6) NULL, + ready_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + deleted_at DATETIME(6) NULL DEFAULT NULL, + INDEX idx_temporary_documents_scope (tenant_id, session_id), + INDEX idx_temporary_documents_status (status), + INDEX idx_temporary_documents_expires (expires_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/scripts/migrate.sh b/scripts/migrate.sh index f11514f85c..c6bd5980a0 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -1,122 +1,290 @@ #!/bin/bash -set -e +set -euo pipefail -# Get the script directory and project root -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -# Load .env file if it exists (for development mode) -if [ -f "$PROJECT_ROOT/.env" ]; then - echo "Loading .env file from $PROJECT_ROOT/.env" +if [ -f "${PROJECT_ROOT}/.env" ]; then + echo "Loading .env file from ${PROJECT_ROOT}/.env" set -a - source "$PROJECT_ROOT/.env" + source "${PROJECT_ROOT}/.env" set +a fi -# Database connection details (can be overridden by environment variables) -DB_HOST=${DB_HOST:-localhost} -DB_PORT=${DB_PORT:-5432} -DB_USER=${DB_USER:-postgres} -DB_PASSWORD=${DB_PASSWORD:-postgres} -DB_NAME=${DB_NAME:-WeKnora} - -# Use versioned migrations directory -MIGRATIONS_DIR="${MIGRATIONS_DIR:-migrations/versioned}" - -# Check if migrate tool is installed -if ! command -v migrate &> /dev/null; then - echo "Error: migrate tool is not installed" - echo "Install it with: go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest" - exit 1 -fi +DB_DRIVER="${DB_DRIVER:-postgres}" +case "${DB_DRIVER}" in + postgres) + DB_PORT_DEFAULT=5432 + DB_USER_DEFAULT=postgres + DB_PASSWORD_DEFAULT=postgres + MIGRATIONS_DIR_DEFAULT="migrations/versioned" + ;; + mysql) + DB_PORT_DEFAULT=3306 + DB_USER_DEFAULT=weknora + DB_PASSWORD_DEFAULT="" + MIGRATIONS_DIR_DEFAULT="migrations/mysql" + ;; + *) + echo "Error: unsupported DB_DRIVER='${DB_DRIVER}' (expected 'postgres' or 'mysql')" >&2 + exit 1 + ;; +esac -# Construct the database URL -# If DB_URL is already set in .env, use it but ensure sslmode=disable is set -# Otherwise, construct it from individual components -if [ -n "$DB_URL" ]; then - # If DB_URL already exists, ensure sslmode=disable is set (unless sslmode is already specified) - if [[ "$DB_URL" != *"sslmode="* ]]; then - # Add sslmode=disable if not present - if [[ "$DB_URL" == *"?"* ]]; then - DB_URL="${DB_URL}&sslmode=disable" - else - DB_URL="${DB_URL}?sslmode=disable" - fi - elif [[ "$DB_URL" == *"sslmode=require"* ]] || [[ "$DB_URL" == *"sslmode=prefer"* ]]; then - # Replace sslmode=require/prefer with sslmode=disable for local dev - DB_URL="${DB_URL//sslmode=require/sslmode=disable}" - DB_URL="${DB_URL//sslmode=prefer/sslmode=disable}" +DB_HOST="${DB_HOST:-localhost}" +DB_PORT="${DB_PORT:-${DB_PORT_DEFAULT}}" +DB_USER="${DB_USER:-${DB_USER_DEFAULT}}" +DB_PASSWORD="${DB_PASSWORD:-${DB_PASSWORD_DEFAULT}}" +DB_NAME="${DB_NAME:-WeKnora}" +DB_URL="${DB_URL:-}" +MIGRATIONS_DIR="${MIGRATIONS_DIR:-${MIGRATIONS_DIR_DEFAULT}}" + +require_migrate() { + if ! command -v migrate >/dev/null 2>&1; then + echo "Error: migrate tool is not installed" >&2 + echo "Install: go install -tags 'postgres mysql' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.19.1" >&2 + exit 1 fi -else - # Use Python to properly URL encode password if it contains special characters - # This handles special characters in passwords correctly - if command -v python3 &> /dev/null; then - ENCODED_PASSWORD=$(python3 -c "import urllib.parse; print(urllib.parse.quote('$DB_PASSWORD', safe=''))") - else - # Fallback: try to use printf for basic encoding (may not work for all special chars) - ENCODED_PASSWORD="$DB_PASSWORD" +} + +build_database_url() { + if ! command -v python3 >/dev/null 2>&1; then + echo "Error: python3 is required to build the database URL safely" >&2 + exit 1 fi - DB_URL="postgres://${DB_USER}:${ENCODED_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable" -fi -# Execute migration based on command -case "$1" in + export DB_DRIVER DB_HOST DB_PORT DB_USER DB_PASSWORD DB_NAME DB_URL + export DB_SSLMODE DB_CONNECT_TIMEOUT DB_READ_TIMEOUT DB_WRITE_TIMEOUT + export DB_USE_TLS DB_TLS_SERVER_NAME DB_TLS_CA DB_TLS_CERT DB_TLS_KEY + export DB_TLS_INSECURE_SKIP_VERIFY + + python3 <<'PY' +import os +import sys +from urllib.parse import parse_qsl, quote, urlencode + + +def env(name, default=""): + value = os.environ.get(name, "").strip() + return value or default + + +def parse_bool(name, default=False): + value = env(name) + if not value: + return default + normalized = value.lower() + if normalized in {"1", "t", "true"}: + return True + if normalized in {"0", "f", "false"}: + return False + raise SystemExit(f"Error: {name} must be true or false, got {value!r}") + + +def normalize_database_url(url, driver): + valid_scheme = ( + driver == "postgres" and url.startswith(("postgres://", "postgresql://")) + ) or (driver == "mysql" and url.startswith("mysql://")) + if not valid_scheme: + raise SystemExit(f"Error: DB_URL scheme does not match DB_DRIVER={driver!r}") + + base, separator, raw_query = url.rpartition("?") + if not separator: + base = url + raw_query = "" + params = parse_qsl(raw_query, keep_blank_values=True) + + if driver == "postgres": + sslmode_indexes = [index for index, (key, _) in enumerate(params) if key == "sslmode"] + if len(sslmode_indexes) > 1: + raise SystemExit("Error: PostgreSQL DB_URL must not repeat sslmode") + if not sslmode_indexes: + params.append(("sslmode", env("DB_SSLMODE", "disable"))) + return f"{base}?{urlencode(params)}" + + def require_parameter(name, expected, *, case_insensitive=False): + indexes = [index for index, (key, _) in enumerate(params) if key == name] + if len(indexes) > 1: + raise SystemExit(f"Error: MySQL DB_URL must not repeat {name}") + if not indexes: + params.append((name, expected)) + return + + index = indexes[0] + value = params[index][1] + matches = value.lower() == expected.lower() if case_insensitive else value == expected + if not matches: + raise SystemExit(f"Error: MySQL DB_URL must set {name}={expected}") + params[index] = (name, expected) + + require_parameter("multiStatements", "true", case_insensitive=True) + require_parameter("time_zone", "'+00:00'") + return f"{base}?{urlencode(params)}" + + +driver = env("DB_DRIVER", "postgres").lower() +database_url = env("DB_URL") +if database_url: + print(normalize_database_url(database_url, driver)) + sys.exit(0) + +host = env("DB_HOST", "localhost") +port = env("DB_PORT", "5432" if driver == "postgres" else "3306") +user = env("DB_USER", "postgres" if driver == "postgres" else "weknora") +password = os.environ.get("DB_PASSWORD", "") +database = env("DB_NAME", "WeKnora") + +if not password: + raise SystemExit("Error: DB_PASSWORD is required when DB_URL is not set") +if not host or not user or not database: + raise SystemExit("Error: DB_HOST, DB_USER, and DB_NAME must be non-empty") +try: + port_number = int(port) +except ValueError as exc: + raise SystemExit(f"Error: DB_PORT must be an integer, got {port!r}") from exc +if not 1 <= port_number <= 65535: + raise SystemExit(f"Error: DB_PORT must be between 1 and 65535, got {port!r}") + +encoded_user = quote(user, safe="") +encoded_password = quote(password, safe="") +encoded_database = quote(database, safe="") +network_host = host +if ":" in network_host and not network_host.startswith("["): + network_host = f"[{network_host}]" + +if driver == "postgres": + sslmode = env("DB_SSLMODE", "disable") + query = urlencode({"sslmode": sslmode}) + print( + f"postgres://{encoded_user}:{encoded_password}@" + f"{network_host}:{port_number}/{encoded_database}?{query}" + ) + sys.exit(0) + +if driver != "mysql": + raise SystemExit(f"Error: unsupported DB_DRIVER={driver!r}") + +params = { + "multiStatements": "true", + "parseTime": "true", + "loc": "UTC", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "time_zone": "'+00:00'", + "timeout": env("DB_CONNECT_TIMEOUT", "10s"), + "readTimeout": env("DB_READ_TIMEOUT", "30s"), + "writeTimeout": env("DB_WRITE_TIMEOUT", "30s"), +} + +use_tls = parse_bool("DB_USE_TLS") +insecure = parse_bool("DB_TLS_INSECURE_SKIP_VERIFY") +server_name = env("DB_TLS_SERVER_NAME") +ca_file = env("DB_TLS_CA") +cert_file = env("DB_TLS_CERT") +key_file = env("DB_TLS_KEY") +has_tls_settings = bool(server_name or ca_file or cert_file or key_file or insecure) + +if not use_tls and has_tls_settings: + raise SystemExit("Error: DB_USE_TLS must be true when DB_TLS_* settings are configured") +if bool(cert_file) != bool(key_file): + raise SystemExit("Error: DB_TLS_CERT and DB_TLS_KEY must be configured together") +if server_name and server_name.strip("[]").casefold() != host.strip("[]").casefold(): + raise SystemExit( + "Error: migrate CLI cannot use DB_TLS_SERVER_NAME different from DB_HOST; " + "provide a DB_URL whose host matches the certificate or use application auto-migration" + ) + +if use_tls: + if ca_file: + params["tls"] = "custom" + params["x-tls-ca"] = ca_file + if cert_file: + params["x-tls-cert"] = cert_file + params["x-tls-key"] = key_file + if insecure: + params["x-tls-insecure-skip-verify"] = "true" + elif cert_file: + raise SystemExit("Error: migrate CLI requires DB_TLS_CA when mTLS certificates are configured") + elif insecure: + params["tls"] = "skip-verify" + else: + params["tls"] = "true" + +query = urlencode(params) +print( + f"mysql://{encoded_user}:{encoded_password}@" + f"tcp({network_host}:{port_number})/{encoded_database}?{query}" +) +PY +} + +run_database_migration() { + require_migrate + local database_url + database_url="$(build_database_url)" + ( + cd "${PROJECT_ROOT}" + migrate -path "${MIGRATIONS_DIR}" -database "${database_url}" "$@" + ) +} + +case "${1:-}" in up) echo "Running migrations up..." - echo "DB_URL: ${DB_URL}" - echo "DB_USER: ${DB_USER}" - echo "DB_PASSWORD: ${DB_PASSWORD}" + echo "DB_DRIVER: ${DB_DRIVER}" echo "DB_HOST: ${DB_HOST}" echo "DB_PORT: ${DB_PORT}" echo "DB_NAME: ${DB_NAME}" echo "MIGRATIONS_DIR: ${MIGRATIONS_DIR}" - migrate -path ${MIGRATIONS_DIR} -database ${DB_URL} up + run_database_migration up ;; down) echo "Running migrations down..." - migrate -path ${MIGRATIONS_DIR} -database ${DB_URL} down - ;; - create) - if [ -z "$2" ]; then - echo "Error: Migration name is required" - echo "Usage: $0 create " - exit 1 - fi - echo "Creating migration files for $2..." - migrate create -ext sql -dir ${MIGRATIONS_DIR} -seq $2 - echo "Created:" - echo " - ${MIGRATIONS_DIR}/$(ls -t ${MIGRATIONS_DIR} | head -1)" - echo " - ${MIGRATIONS_DIR}/$(ls -t ${MIGRATIONS_DIR} | head -2 | tail -1)" + run_database_migration down ;; version) echo "Checking current migration version..." - migrate -path ${MIGRATIONS_DIR} -database ${DB_URL} version + run_database_migration version ;; force) - if [ -z "$2" ]; then - echo "Error: Version number is required" - echo "Usage: $0 force " - echo "Note: Use -1 to reset to no version (allows re-running all migrations)" + if [ -z "${2:-}" ]; then + echo "Error: Version number is required" >&2 + echo "Usage: $0 force " >&2 exit 1 fi - VERSION="$2" - echo "Forcing migration version to $VERSION..." - # Use env to pass the command, avoiding shell flag parsing issues with negative numbers - env migrate -path "${MIGRATIONS_DIR}" -database "${DB_URL}" force -- "$VERSION" + echo "Forcing migration version to ${2}..." + run_database_migration force -- "${2}" ;; goto) - if [ -z "$2" ]; then - echo "Error: Version number is required" - echo "Usage: $0 goto " + if [ -z "${2:-}" ]; then + echo "Error: Version number is required" >&2 + echo "Usage: $0 goto " >&2 + exit 1 + fi + echo "Migrating to version ${2}..." + run_database_migration goto "${2}" + ;; + create) + if [ -z "${2:-}" ]; then + echo "Error: Migration name is required" >&2 + echo "Usage: $0 create " >&2 + exit 1 + fi + if [ "${DB_DRIVER}" = "mysql" ]; then + echo "Error: MySQL schema changes must be folded into migrations/mysql/000000_init.up.sql and its matching down baseline" >&2 exit 1 fi - echo "Migrating to version $2..." - migrate -path ${MIGRATIONS_DIR} -database ${DB_URL} goto $2 + require_migrate + echo "Creating migration files for ${2}..." + ( + cd "${PROJECT_ROOT}" + migrate create -ext sql -dir "${MIGRATIONS_DIR}" -seq "${2}" + ) ;; *) - echo "Usage: $0 {up|down|create |version|force |goto }" + echo "Usage: $0 {up|down|version|force |goto |create }" >&2 exit 1 ;; esac -echo "Migration command completed successfully" \ No newline at end of file +echo "Migration command completed successfully"