From 41a439d83029bb3465c5043798fb59add50a9c55 Mon Sep 17 00:00:00 2001 From: SmartUp Developer Date: Mon, 25 May 2026 00:08:10 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20SQLite=20WAL=20+=20=E5=A4=8D=E5=90=88?= =?UTF-8?q?=E7=B4=A2=E5=BC=95=20+=20GZip=20+=20scheduler=20jitter=20+=20?= =?UTF-8?q?=E6=9E=84=E5=BB=BA=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 6 ++-- backend/app/database.py | 41 +++++++++++++++++++++++++- backend/app/main.py | 2 ++ backend/app/models/notification_log.py | 7 ++++- backend/app/models/snapshot.py | 6 +++- backend/app/models/upstream.py | 6 +++- backend/app/models/upstream_key.py | 3 +- backend/app/models/website.py | 7 ++++- backend/app/services/scheduler.py | 2 ++ docker-compose.yml | 5 ++++ 10 files changed, 77 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1471569..d2b17c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1 +# syntax=docker/dockerfile:1.4 # ---- Stage 1: Build frontend ---- FROM node:20-alpine AS frontend-build WORKDIR /frontend @@ -42,7 +42,9 @@ RUN --mount=type=cache,target=/root/.cache/pip \ -r requirements.txt # Playwright Chromium：安装在镜像层中，业务代码变更不会触发重下 -RUN playwright install chromium +# --mount=type=cache 利用 BuildKit 缓存避免每次构建重下 ~170 MB 浏览器 +RUN --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked \ + playwright install chromium # 源码层：业务代码变更不影响上面所有依赖层 COPY backend/ . diff --git a/backend/app/database.py b/backend/app/database.py index 90f0ef2..0d10afd 100644 --- a/backend/app/database.py +++ b/backend/app/database.py @@ -1,4 +1,4 @@ -from sqlalchemy import create_engine, inspect, text +from sqlalchemy import create_engine, event, inspect, text from sqlalchemy.orm import sessionmaker, DeclarativeBase from app.config import get_settings @@ -8,6 +8,21 @@ engine = create_engine( settings.database_url, connect_args={"check_same_thread": False}, ) + +# ── SQLite 性能 PRAGMA（WAL + 缓存 + 超时） ── +@event.listens_for(engine, "connect", insert=True) +def _set_sqlite_pragma(dbapi_connection, connection_record): + if engine.dialect.name != "sqlite": + return + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA journal_mode=WAL;") + cursor.execute("PRAGMA synchronous=NORMAL;") + cursor.execute("PRAGMA busy_timeout=5000;") + cursor.execute("PRAGMA foreign_keys=ON;") + cursor.execute("PRAGMA cache_size=-20000;") # 20 MB page cache + cursor.execute("PRAGMA mmap_size=67108864;") # 64 MB（小内存容器友好） + cursor.close() + SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) @@ -28,11 +43,35 @@ def init_db(): # import models so SQLAlchemy registers them from app.models import admin_user, upstream, snapshot, webhook_config, notification_log, custom_page, website, revoked_token, upstream_key # noqa: F401 Base.metadata.create_all(bind=engine) + _ensure_indexes() _migrate_custom_pages() _migrate_upstreams() _migrate_upstream_generated_keys() +# ── 已有数据库幂等索引迁移 ───────────────────────────────── +_NEW_INDEXES = [ + "CREATE INDEX IF NOT EXISTS ix_snapshot_upstream_captured ON upstream_rate_snapshots(upstream_id, captured_at DESC)", + "CREATE INDEX IF NOT EXISTS ix_notif_event_created ON notification_logs(event_type, created_at DESC)", + "CREATE INDEX IF NOT EXISTS ix_notif_status_created ON notification_logs(status, created_at DESC)", + "CREATE INDEX IF NOT EXISTS ix_sync_website_created ON website_sync_logs(website_id, created_at DESC)", + "CREATE INDEX IF NOT EXISTS ix_sync_binding_created ON website_sync_logs(binding_id, created_at DESC)", + "CREATE INDEX IF NOT EXISTS ix_upstream_enabled ON upstreams(enabled)", + "CREATE INDEX IF NOT EXISTS ix_key_upstream_name ON upstream_generated_keys(upstream_id, key_name)", +] + + +def _ensure_indexes() -> None: + """创建 InitDB 时可能未包含的复合索引，幂等执行。""" + with engine.begin() as conn: + for stmt in _NEW_INDEXES: + try: + conn.execute(text(stmt)) + except Exception: + logger = __import__("logging").getLogger(__name__) + logger.warning("index creation failed (non-fatal): %s", stmt[:60]) + + def _migrate_custom_pages(): """Apply small SQLite-safe migrations for deployments without Alembic.""" inspector = inspect(engine) diff --git a/backend/app/main.py b/backend/app/main.py index 08ec5c3..ea004d4 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -5,6 +5,7 @@ from pathlib import Path from fastapi import FastAPI, HTTPException from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.gzip import GZipMiddleware from fastapi.staticfiles import StaticFiles from fastapi.responses import FileResponse @@ -88,6 +89,7 @@ app.add_middleware( allow_methods=["*"], allow_headers=["*"], ) +app.add_middleware(GZipMiddleware, minimum_size=1024) # API routers app.include_router(auth.router) diff --git a/backend/app/models/notification_log.py b/backend/app/models/notification_log.py index 01f40f9..c189f21 100644 --- a/backend/app/models/notification_log.py +++ b/backend/app/models/notification_log.py @@ -1,6 +1,6 @@ from datetime import datetime, timezone from typing import Optional -from sqlalchemy import Integer, String, DateTime, Text, ForeignKey +from sqlalchemy import Index, Integer, String, DateTime, Text, ForeignKey, text from sqlalchemy.orm import mapped_column, Mapped from app.database import Base @@ -19,3 +19,8 @@ class NotificationLog(Base): status: Mapped[str] = mapped_column(String(16), nullable=False) response_text: Mapped[Optional[str]] = mapped_column(Text, nullable=True) created_at: Mapped[datetime] = mapped_column(DateTime, default=lambda: datetime.now(timezone.utc), index=True) + + __table_args__ = ( + Index("ix_notif_event_created", "event_type", text("created_at DESC")), + Index("ix_notif_status_created", "status", text("created_at DESC")), + ) diff --git a/backend/app/models/snapshot.py b/backend/app/models/snapshot.py index b24a7be..e32dbcd 100644 --- a/backend/app/models/snapshot.py +++ b/backend/app/models/snapshot.py @@ -1,5 +1,5 @@ from datetime import datetime, timezone -from sqlalchemy import Integer, Text, DateTime, ForeignKey +from sqlalchemy import Index, Integer, Text, DateTime, ForeignKey, text from sqlalchemy.orm import mapped_column, Mapped from app.database import Base @@ -11,3 +11,7 @@ class UpstreamRateSnapshot(Base): upstream_id: Mapped[int] = mapped_column(Integer, ForeignKey("upstreams.id", ondelete="CASCADE"), index=True) snapshot_json: Mapped[str] = mapped_column(Text, nullable=False) captured_at: Mapped[datetime] = mapped_column(DateTime, default=lambda: datetime.now(timezone.utc), index=True) + + __table_args__ = ( + Index("ix_snapshot_upstream_captured", "upstream_id", text("captured_at DESC")), + ) diff --git a/backend/app/models/upstream.py b/backend/app/models/upstream.py index ddddd2b..d41130a 100644 --- a/backend/app/models/upstream.py +++ b/backend/app/models/upstream.py @@ -1,6 +1,6 @@ from datetime import datetime, timezone from typing import Optional -from sqlalchemy import Integer, String, Boolean, DateTime, Text, Float +from sqlalchemy import Index, Integer, String, Boolean, DateTime, Text, Float from sqlalchemy.orm import mapped_column, Mapped from app.database import Base @@ -36,3 +36,7 @@ class Upstream(Base): updated_at: Mapped[datetime] = mapped_column( DateTime, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc) ) + + __table_args__ = ( + Index("ix_upstream_enabled", "enabled"), + ) diff --git a/backend/app/models/upstream_key.py b/backend/app/models/upstream_key.py index b7fb7e2..2d414a2 100644 --- a/backend/app/models/upstream_key.py +++ b/backend/app/models/upstream_key.py @@ -1,7 +1,7 @@ from datetime import datetime, timezone from typing import Optional -from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, UniqueConstraint +from sqlalchemy import DateTime, ForeignKey, Index, Integer, String, Text, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column from app.database import Base @@ -30,4 +30,5 @@ class UpstreamGeneratedKey(Base): __table_args__ = ( UniqueConstraint("upstream_id", "group_id", "key_name", name="uq_upstream_group_key"), + Index("ix_key_upstream_name", "upstream_id", "key_name"), ) diff --git a/backend/app/models/website.py b/backend/app/models/website.py index c1afad5..24d5cf3 100644 --- a/backend/app/models/website.py +++ b/backend/app/models/website.py @@ -1,7 +1,7 @@ from datetime import datetime, timezone from typing import Optional -from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, String, Text +from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, String, Text, text from sqlalchemy.orm import Mapped, mapped_column from app.database import Base @@ -66,3 +66,8 @@ class WebsiteSyncLog(Base): status: Mapped[str] = mapped_column(String(16), nullable=False) message: Mapped[str] = mapped_column(Text, default="") created_at: Mapped[datetime] = mapped_column(DateTime, default=lambda: datetime.now(timezone.utc), index=True) + + __table_args__ = ( + Index("ix_sync_website_created", "website_id", text("created_at DESC")), + Index("ix_sync_binding_created", "binding_id", text("created_at DESC")), + ) diff --git a/backend/app/services/scheduler.py b/backend/app/services/scheduler.py index 8de6254..8fa97c6 100644 --- a/backend/app/services/scheduler.py +++ b/backend/app/services/scheduler.py @@ -284,6 +284,8 @@ def refresh_upstream(upstream_id: int, interval_seconds: int = 0, enabled: bool replace_existing=True, coalesce=True, max_instances=1, + misfire_grace_time=60, + jitter=30, ) logger.info("scheduler job %s set to %ds interval", job_id, interval_seconds) diff --git a/docker-compose.yml b/docker-compose.yml index 0400847..816d8f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,11 @@ services: - DATABASE_URL=sqlite:////app/data/app.db - TZ=${TZ:-Asia/Shanghai} - UNHEALTHY_THRESHOLD=${UNHEALTHY_THRESHOLD:-3} + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8000/healthz"] interval: 30s