LiuMangMang/SmartUp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 重构接口日志页面与接口监控看板，支持HTTP方法过滤和实时轮询

Browse Source
This commit is contained in:
liumangmang
2026-07-01 15:04:32 +08:00
parent 397a14c978
commit 8abfa4c5ef
19 changed files with 2463 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/config.py
+3
View File
@@ -15,6 +15,9 @@ class Settings(BaseSettings):
tz: str = "Asia/Shanghai"
# consecutive failures before upstream goes unhealthy
unhealthy_threshold: int = 3
# external API log retention
external_api_log_retention_days: int = 30
external_api_log_slow_ms: int = 3000
@property
def cors_origin_list(self) -> list[str]:
backend/app/database.py
+3 -1
View File
@@ -41,7 +41,7 @@ def get_db():
def init_db():
"""Create all tables."""
# import models so SQLAlchemy registers them
from app.models import admin_user, upstream, snapshot, webhook_config, notification_log, custom_page, website, revoked_token, upstream_key # noqa: F401
from app.models import admin_user, upstream, snapshot, webhook_config, notification_log, custom_page, website, revoked_token, upstream_key, external_api_log # noqa: F401
Base.metadata.create_all(bind=engine)
_ensure_indexes()
_migrate_custom_pages()
@@ -58,6 +58,8 @@ _NEW_INDEXES = [
"CREATE INDEX IF NOT EXISTS ix_sync_binding_created ON website_sync_logs(binding_id, created_at DESC)",
"CREATE INDEX IF NOT EXISTS ix_upstream_enabled ON upstreams(enabled)",
"CREATE INDEX IF NOT EXISTS ix_key_upstream_name ON upstream_generated_keys(upstream_id, key_name)",
"CREATE INDEX IF NOT EXISTS ix_external_api_logs_created_at_desc ON external_api_logs(created_at DESC)",
"CREATE INDEX IF NOT EXISTS ix_external_api_logs_direction_created ON external_api_logs(direction, created_at DESC)",
]
backend/app/main.py
+2 -1
View File
@@ -15,7 +15,7 @@ from app.models.admin_user import AdminUser
from app.database import SessionLocal
from app.utils.auth import hash_password, verify_password, validate_password_supported
from app.services.scheduler import start_scheduler, stop_scheduler
from app.routers import auth, upstreams, webhooks, logs, custom_pages, websites, auth_capture
from app.routers import auth, upstreams, webhooks, logs, custom_pages, websites, auth_capture, external_api_logs
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
logger = logging.getLogger(__name__)
@@ -97,6 +97,7 @@ app.include_router(logs.router)
app.include_router(custom_pages.router)
app.include_router(websites.router)
app.include_router(auth_capture.router)
app.include_router(external_api_logs.router)
@app.get("/healthz")
backend/app/models/__init__.py
+1
View File
@@ -1 +1,2 @@
# models package
from app.models.external_api_log import ExternalApiLog
backend/app/models/external_api_log.py
+28
View File
@@ -0,0 +1,28 @@
from datetime import datetime, timezone
from typing import Optional
from sqlalchemy import Index, Integer, String, DateTime, Boolean, text
from sqlalchemy.orm import mapped_column, Mapped
from app.database import Base
class ExternalApiLog(Base):
__tablename__ = "external_api_logs"
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
direction: Mapped[str] = mapped_column(String(32), index=True) # "upstream" | "website"
target_type: Mapped[str] = mapped_column(String(32), index=True) # "upstream" | "website"
target_id: Mapped[Optional[int]] = mapped_column(Integer, nullable=True, index=True)
target_name: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
method: Mapped[str] = mapped_column(String(16))
path: Mapped[str] = mapped_column(String(1024))
url_host: Mapped[str] = mapped_column(String(255))
status_code: Mapped[Optional[int]] = mapped_column(Integer, nullable=True, index=True)
success: Mapped[bool] = mapped_column(Boolean, default=True, index=True)
duration_ms: Mapped[int] = mapped_column(Integer)
error_type: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
error_message: Mapped[Optional[str]] = mapped_column(String(1024), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime, default=lambda: datetime.now(timezone.utc), index=True)
__table_args__ = (
Index("ix_external_api_logs_created_at_desc", text("created_at DESC")),
)
backend/app/routers/external_api_logs.py
+161
View File
@@ -0,0 +1,161 @@
"""External API logs listing with filters and summary."""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from typing import List, Optional
from fastapi import APIRouter, Depends, Query
from sqlalchemy import cast, case, func, Integer
from sqlalchemy.orm import Session
from app.database import get_db
from app.models.external_api_log import ExternalApiLog
from app.schemas.external_api_log import ExternalApiLogResponse, ExternalApiLogSummaryItem
from app.utils.auth import get_current_user
from app.config import get_settings
router = APIRouter(prefix="/api/external-api-logs", tags=["external_api_logs"])
@router.get("", response_model=List[ExternalApiLogResponse])
def list_external_api_logs(
direction: Optional[str] = Query(None),
target_type: Optional[str] = Query(None),
target_id: Optional[int] = Query(None),
method: Optional[str] = Query(None),
success: Optional[bool] = Query(None),
status_code: Optional[int] = Query(None),
path: Optional[str] = Query(None),
limit: int = Query(100, le=500),
offset: int = Query(0),
db: Session = Depends(get_db),
_=Depends(get_current_user),
):
q = db.query(ExternalApiLog)
if direction:
q = q.filter(ExternalApiLog.direction == direction)
if target_type:
q = q.filter(ExternalApiLog.target_type == target_type)
if target_id is not None:
q = q.filter(ExternalApiLog.target_id == target_id)
if method and isinstance(method, str):
q = q.filter(ExternalApiLog.method == method.upper())
if success is not None:
q = q.filter(ExternalApiLog.success == success)
if status_code is not None:
q = q.filter(ExternalApiLog.status_code == status_code)
if path:
q = q.filter(ExternalApiLog.path.like(f"%{path}%"))
logs = q.order_by(ExternalApiLog.created_at.desc()).offset(offset).limit(limit).all()
return logs
@router.get("/count")
def count_external_api_logs(
direction: Optional[str] = Query(None),
target_type: Optional[str] = Query(None),
target_id: Optional[int] = Query(None),
method: Optional[str] = Query(None),
success: Optional[bool] = Query(None),
status_code: Optional[int] = Query(None),
path: Optional[str] = Query(None),
db: Session = Depends(get_db),
_=Depends(get_current_user),
):
q = db.query(ExternalApiLog)
if direction:
q = q.filter(ExternalApiLog.direction == direction)
if target_type:
q = q.filter(ExternalApiLog.target_type == target_type)
if target_id is not None:
q = q.filter(ExternalApiLog.target_id == target_id)
if method and isinstance(method, str):
q = q.filter(ExternalApiLog.method == method.upper())
if success is not None:
q = q.filter(ExternalApiLog.success == success)
if status_code is not None:
q = q.filter(ExternalApiLog.status_code == status_code)
if path:
q = q.filter(ExternalApiLog.path.like(f"%{path}%"))
return {"total": q.count()}
@router.get("/summary", response_model=List[ExternalApiLogSummaryItem])
def get_external_api_log_summary(
slow_ms: Optional[int] = Query(None),
hours: int = Query(24, ge=1, le=720),
limit: int = Query(50, ge=1, le=200),
db: Session = Depends(get_db),
_=Depends(get_current_user),
):
"""Return aggregated summary grouped by direction + target + method + path.
- hours: time window (default 24, max 720). None means no time filter.
- limit: max rows returned (default 50, max 200).
"""
settings = get_settings()
slow_threshold = slow_ms if slow_ms is not None else settings.external_api_log_slow_ms
q = db.query(*_summary_cols(slow_threshold))
cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
q = q.filter(ExternalApiLog.created_at >= cutoff)
q = q.group_by(
ExternalApiLog.direction,
ExternalApiLog.target_type,
ExternalApiLog.target_id,
ExternalApiLog.target_name,
ExternalApiLog.method,
ExternalApiLog.path,
).order_by(func.sum(ExternalApiLog.duration_ms).desc())
q = q.limit(limit)
rows = q.all()
return [
ExternalApiLogSummaryItem(
direction=row.direction,
target_type=row.target_type,
target_id=row.target_id,
target_name=row.target_name,
method=row.method,
path=row.path,
call_count=row.call_count,
success_count=row.success_count or 0,
fail_count=row.fail_count or 0,
avg_duration_ms=round(float(row.avg_duration_ms or 0), 1),
max_duration_ms=row.max_duration_ms or 0,
slow_count=row.slow_count or 0,
)
for row in rows
]
def _summary_cols(slow_threshold: int) -> list:
return [
ExternalApiLog.direction,
ExternalApiLog.target_type,
ExternalApiLog.target_id,
ExternalApiLog.target_name,
ExternalApiLog.method,
ExternalApiLog.path,
func.count(ExternalApiLog.id).label("call_count"),
func.sum(cast(ExternalApiLog.success, Integer)).label("success_count"),
func.sum(cast(1 - cast(ExternalApiLog.success, Integer), Integer)).label("fail_count"),
func.avg(ExternalApiLog.duration_ms).label("avg_duration_ms"),
func.max(ExternalApiLog.duration_ms).label("max_duration_ms"),
func.sum(case((ExternalApiLog.duration_ms >= slow_threshold, 1), else_=0)).label("slow_count"),
]
def clean_expired_logs(db: Session) -> int:
"""Delete external API logs older than the configured retention period.
Returns the number of deleted rows.
"""
settings = get_settings()
cutoff = datetime.now(timezone.utc) - timedelta(days=settings.external_api_log_retention_days)
deleted = db.query(ExternalApiLog).filter(
ExternalApiLog.created_at < cutoff
).delete(synchronize_session=False)
if deleted:
db.commit()
return deleted
backend/app/routers/upstreams.py
+8
View File
@@ -195,6 +195,8 @@ def list_generated_keys(uid: int, db: Session = Depends(get_db), _=Depends(get_c
auth_config=auth_config,
timeout=float(upstream.timeout_seconds),
on_auth_config_update=lambda updated: _persist_auth_config_update(db, upstream, updated),
target_id=upstream.id,
target_name=upstream.name,
) as client:
client.login()
for prefix in website_sync._fetch_remote_managed_prefixes(db, uid):
@@ -628,6 +630,8 @@ def generate_keys_by_groups(
auth_config=auth_config,
timeout=float(u.timeout_seconds),
on_auth_config_update=lambda updated: _persist_auth_config_update(db, u, updated),
target_id=u.id,
target_name=u.name,
) as client:
try:
client.login()
@@ -679,6 +683,8 @@ def _test_upstream_core(db: Session, u: Upstream) -> UpstreamBatchActionItem:
auth_config=auth_config,
timeout=float(u.timeout_seconds),
on_auth_config_update=lambda updated: _persist_auth_config_update(db, u, updated),
target_id=u.id,
target_name=u.name,
) as client:
client.login()
groups = client.get_available_groups(u.groups_endpoint)
@@ -719,6 +725,8 @@ def _check_now_core(db: Session, u: Upstream) -> tuple[str, bool]:
auth_config=auth_config,
timeout=float(u.timeout_seconds),
on_auth_config_update=lambda updated: _persist_auth_config_update(db, u, updated),
target_id=u.id,
target_name=u.name,
) as client:
client.login()
groups = client.get_available_groups(u.groups_endpoint)
backend/app/routers/websites.py
+2
View File
@@ -140,6 +140,8 @@ def _client(row: Website) -> Sub2ApiWebsiteClient:
auth_type=row.auth_type,
auth_config=json.loads(row.auth_config_json or "{}"),
timeout=float(row.timeout_seconds),
target_id=row.id,
target_name=row.name,
)
backend/app/schemas/external_api_log.py
+37
View File
@@ -0,0 +1,37 @@
from datetime import datetime
from typing import List, Optional
from pydantic import BaseModel
class ExternalApiLogResponse(BaseModel):
id: int
direction: str
target_type: str
target_id: Optional[int] = None
target_name: Optional[str] = None
method: str
path: str
url_host: str
status_code: Optional[int] = None
success: bool
duration_ms: int
error_type: Optional[str] = None
error_message: Optional[str] = None
created_at: datetime
model_config = {"from_attributes": True}
class ExternalApiLogSummaryItem(BaseModel):
direction: str
target_type: str
target_id: Optional[int] = None
target_name: Optional[str] = None
method: str
path: str
call_count: int
success_count: int
fail_count: int
avg_duration_ms: float
max_duration_ms: int
slow_count: int
backend/app/services/external_api_logger.py
+74
View File
@@ -0,0 +1,74 @@
"""Log external API calls to the database and console."""
from __future__ import annotations
import logging
from datetime import datetime, timezone
from typing import Optional
from app.database import SessionLocal
from app.models.external_api_log import ExternalApiLog
logger = logging.getLogger(__name__)
def log_external_api_call(
direction: str,
target_type: str,
method: str,
path: str,
url_host: str,
duration_ms: int,
status_code: Optional[int] = None,
success: bool = True,
target_id: Optional[int] = None,
target_name: Optional[str] = None,
error_type: Optional[str] = None,
error_message: Optional[str] = None,
) -> None:
"""Persist an external API call log entry and write a structured console log.
No request/response bodies, tokens, cookies, or API keys are recorded.
"""
# Console log (structured, for Docker log aggregation)
logger.info(
"api_log direction=%s target_type=%s target_id=%s target_name=%s "
"method=%s path=%s url_host=%s status_code=%s success=%s "
"duration_ms=%d error_type=%s error_message=%s",
direction,
target_type,
target_id,
target_name or "",
method,
path,
url_host,
status_code or "",
success,
duration_ms,
error_type or "",
(error_message or "")[:200],
)
# DB persistence
db = SessionLocal()
try:
log = ExternalApiLog(
direction=direction,
target_type=target_type,
target_id=target_id,
target_name=target_name,
method=method,
path=path,
url_host=url_host,
status_code=status_code,
success=success,
duration_ms=duration_ms,
error_type=error_type,
error_message=error_message,
created_at=datetime.now(timezone.utc),
)
db.add(log)
db.commit()
except Exception:
logger.exception("failed to persist external API log entry")
finally:
db.close()
backend/app/services/scheduler.py
+29
View File
@@ -19,6 +19,7 @@ from app.services.snapshot_service import diff_snapshots, prune_snapshots
from app.services import webhook_service
from app.services import website_sync
from app.config import get_settings
from app.routers.external_api_logs import clean_expired_logs
logger = logging.getLogger(__name__)
@@ -77,6 +78,8 @@ def _check_upstream(upstream_id: int) -> None:
auth_config=auth_config,
timeout=float(upstream.timeout_seconds),
on_auth_config_update=lambda updated: _persist_auth_config_update(upstream, updated),
target_id=upstream.id,
target_name=upstream.name,
) as client:
try:
client.login()
@@ -258,6 +261,8 @@ def _sync_upstream_keys(upstream_id: int, snapshot: dict[str, Any], captured_at:
auth_config=auth_config,
timeout=float(upstream.timeout_seconds),
on_auth_config_update=lambda updated: _persist_auth_config_update(upstream, updated),
target_id=upstream.id,
target_name=upstream.name,
) as client:
client.login()
remote_key_ids = website_sync._fetch_remote_managed_key_ids(db, client, upstream_id)
@@ -328,11 +333,35 @@ def start_scheduler() -> None:
upstreams = db.query(Upstream).filter(Upstream.enabled == True).all()
for u in upstreams:
refresh_upstream(u.id, u.check_interval_seconds, u.enabled)
# Daily cleanup of expired external API logs
_scheduler.add_job(
_cleanup_external_logs,
"interval",
hours=24,
id="cleanup_external_api_logs",
replace_existing=True,
coalesce=True,
max_instances=1,
misfire_grace_time=3600,
)
logger.info("scheduler started with %d upstream job(s)", len(upstreams))
finally:
db.close()
def _cleanup_external_logs() -> None:
"""Delete external API logs older than the configured retention period."""
db = SessionLocal()
try:
deleted = clean_expired_logs(db)
if deleted:
logger.info("cleaned %d expired external API log(s)", deleted)
except Exception:
logger.exception("failed to clean expired external API logs")
finally:
db.close()
def stop_scheduler() -> None:
if _scheduler.running:
_scheduler.shutdown(wait=True)
backend/app/services/upstream_client.py
+61 -20
View File
@@ -3,13 +3,18 @@ from __future__ import annotations
import hashlib
import json
import logging
import re
import time
from typing import Any, Callable, Optional
from urllib.parse import urljoin
from urllib.parse import urljoin, urlparse
import httpx
from app.utils.number import decimal_string
from app.services.external_api_logger import log_external_api_call
logger = logging.getLogger(__name__)
class UpstreamError(RuntimeError):
@@ -429,6 +434,8 @@ class UpstreamClient:
auth_config: dict[str, Any],
timeout: float = 30.0,
on_auth_config_update: Callable[[dict[str, Any]], None] | None = None,
target_id: int | None = None,
target_name: str | None = None,
) -> None:
self.base_url = base_url.rstrip("/")
self.api_prefix = api_prefix.strip("/")
@@ -436,6 +443,8 @@ class UpstreamClient:
self.auth_config = auth_config
self.timeout = timeout
self.on_auth_config_update = on_auth_config_update
self.target_id = target_id
self.target_name = target_name
self._token: str = ""
self._cookies: dict[str, str] = {}
self._new_api_user: str = ""
@@ -549,6 +558,51 @@ class UpstreamClient:
if changed and self.on_auth_config_update:
self.on_auth_config_update(dict(self.auth_config))
def _do_request(self, method: str, url: str, **kwargs: Any) -> httpx.Response:
"""Wrapped self._client.request() with timing and external-API logging.
Every HTTP call (initial, retry, refresh token) goes through here so
the log is accurate and complete.
A response with status_code >= 400 is logged as a failure even though
no exception was raised, so callers can check success/failure rates
accurately. Callers still use raise_for_status() for flow control.
"""
started = time.monotonic()
status_code: int | None = None
error_type: str | None = None
error_msg: str | None = None
try:
resp = self._client.request(method, url, **kwargs)
status_code = getattr(resp, "status_code", None)
if status_code is not None and status_code >= 400:
error_type = "HTTPStatus"
error_msg = f"HTTP {status_code}"
return resp
except Exception as exc:
error_type = type(exc).__name__
error_msg = str(exc)[:500]
if hasattr(exc, "response") and hasattr(exc.response, "status_code"):
status_code = exc.response.status_code
raise
finally:
elapsed = int((time.monotonic() - started) * 1000)
parsed = urlparse(url)
log_external_api_call(
direction="upstream",
target_type="upstream",
method=method,
path=parsed.path or "",
url_host=parsed.hostname or "",
duration_ms=elapsed,
status_code=status_code,
success=error_type is None,
target_id=self.target_id,
target_name=self.target_name,
error_type=error_type,
error_message=error_msg,
)
def _refresh_sub2api_bearer_token(self) -> bool:
if not self._is_sub2api_bearer():
return False
@@ -556,7 +610,7 @@ class UpstreamClient:
if not refresh_token:
return False
try:
resp = self._client.request(
resp = self._do_request(
"POST",
self._url("/auth/refresh"),
json={"refresh_token": refresh_token},
@@ -587,7 +641,7 @@ class UpstreamClient:
allow_refresh: bool = True,
**kwargs: Any,
) -> httpx.Response:
resp = self._client.request(
resp = self._do_request(
method,
url,
headers=self._headers(auth),
@@ -602,7 +656,7 @@ class UpstreamClient:
and self._is_sub2api_bearer()
and self._refresh_sub2api_bearer_token()
):
resp = self._client.request(
resp = self._do_request(
method,
url,
headers=self._headers(auth),
@@ -621,18 +675,9 @@ class UpstreamClient:
raise UpstreamError("Nox-API endpoint requires Nox-Api-User; please fill user id and retry")
url = self._url(path)
if body is not None:
resp = self._send_request(
method,
url,
json=body,
auth=auth,
)
resp = self._send_request(method, url, json=body, auth=auth)
else:
resp = self._send_request(
method,
url,
auth=auth,
)
resp = self._send_request(method, url, auth=auth)
resp.raise_for_status()
ct = resp.headers.get("content-type", "")
if not resp.content:
@@ -689,11 +734,7 @@ class UpstreamClient:
return items, meta
def _request_new_api_token_list(self, path: str, params: dict[str, Any]) -> tuple[list[dict[str, Any]], dict[str, Any]]:
resp = self._send_request(
"GET",
self._url(path),
params=params,
)
resp = self._send_request("GET", self._url(path), params=params)
resp.raise_for_status()
data = resp.json()
self._ensure_api_success(data, "list New-API tokens")
backend/app/services/website_client.py
+64 -15
View File
@@ -1,13 +1,16 @@
from __future__ import annotations
import logging
import sys
import time
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
from typing import Any
from urllib.parse import quote
from urllib.parse import quote, urlparse
import httpx
from app.utils.number import fixed_decimal_number, fixed_decimal_string
from app.services.external_api_logger import log_external_api_call
logger = logging.getLogger(__name__)
@@ -136,12 +139,16 @@ class Sub2ApiWebsiteClient:
auth_type: str,
auth_config: dict[str, Any],
timeout: float = 30.0,
target_id: int | None = None,
target_name: str | None = None,
) -> None:
self.base_url = base_url.rstrip("/")
self.api_prefix = api_prefix.strip("/")
self.auth_type = auth_type
self.auth_config = auth_config
self.timeout = timeout
self.target_id = target_id
self.target_name = target_name
self._client = httpx.Client(timeout=timeout)
def close(self) -> None:
@@ -171,21 +178,63 @@ class Sub2ApiWebsiteClient:
return headers
def _request(self, method: str, path: str, body: Any = None) -> Any:
url = self._url(path)
started = time.monotonic()
status_code: int | None = None
error_type: str | None = None
error_msg: str | None = None
try:
resp = self._client.request(method, self._url(path), json=body, headers=self._headers())
resp.raise_for_status()
except httpx.HTTPStatusError as exc:
raise WebsiteError(_friendly_http_error(exc)) from exc
except httpx.TimeoutException as exc:
raise WebsiteError(_friendly_connection_error(exc)) from exc
except httpx.ConnectError as exc:
raise WebsiteError(_friendly_connection_error(exc)) from exc
if not resp.content:
return None
text = resp.text
if "application/json" not in resp.headers.get("content-type", "") and text.lstrip().startswith("<"):
raise WebsiteError(f"{method} {path} 返回了 HTML，请检查接口地址是否正确")
return resp.json()
try:
resp = self._client.request(method, url, json=body, headers=self._headers())
except httpx.TimeoutException as exc:
error_type, error_msg = type(exc).__name__, str(exc)[:500]
raise WebsiteError(_friendly_connection_error(exc)) from exc
except httpx.ConnectError as exc:
error_type, error_msg = type(exc).__name__, str(exc)[:500]
raise WebsiteError(_friendly_connection_error(exc)) from exc
except httpx.HTTPStatusError as exc:
error_type, error_msg = type(exc).__name__, str(exc)[:500]
status_code = exc.response.status_code
raise WebsiteError(_friendly_http_error(exc)) from exc
status_code = getattr(resp, "status_code", None)
try:
resp.raise_for_status()
except httpx.HTTPStatusError as exc:
error_type, error_msg = type(exc).__name__, str(exc)[:500]
status_code = exc.response.status_code
raise WebsiteError(_friendly_http_error(exc)) from exc
if not resp.content:
return None
text = resp.text
if "application/json" not in resp.headers.get("content-type", "") and text.lstrip().startswith("<"):
error_type = "WebsiteError"
error_msg = f"{method} {path} returned HTML"
raise WebsiteError(f"{method} {path} 返回了 HTML，请检查接口地址是否正确")
return resp.json()
except WebsiteError:
if error_type is None:
error_type = "WebsiteError"
e = sys.exc_info()[1]
if e:
error_msg = str(e)[:500]
raise
finally:
elapsed = int((time.monotonic() - started) * 1000)
parsed = urlparse(url)
log_external_api_call(
direction="website",
target_type="website",
method=method,
path=parsed.path or path,
url_host=parsed.hostname or "",
duration_ms=elapsed,
status_code=status_code,
success=error_type is None,
target_id=self.target_id,
target_name=self.target_name,
error_type=error_type,
error_message=error_msg,
)
def get_groups(self, endpoint: str = "/groups") -> list[dict[str, Any]]:
"""拉取分组列表，尝试 endpoint 和 fallback /groups/all。"""
backend/app/services/website_sync.py
+6
View File
@@ -93,6 +93,8 @@ def _client_for(website: Website) -> Sub2ApiWebsiteClient:
auth_type=website.auth_type,
auth_config=json.loads(website.auth_config_json or "{}"),
timeout=float(website.timeout_seconds),
target_id=website.id,
target_name=website.name,
)
@@ -325,6 +327,8 @@ def _backfill_missing_target_groups_from_remote_accounts(
auth_type=website.auth_type,
auth_config=json.loads(website.auth_config_json or "{}"),
timeout=float(website.timeout_seconds),
target_id=website.id,
target_name=website.name,
) as client:
accounts = client.list_accounts() or []
except Exception as exc:
@@ -838,6 +842,8 @@ def reconcile_upstream_keys_full(db: Session, upstream_id: int) -> bool:
auth_config=auth_config,
timeout=float(upstream.timeout_seconds),
on_auth_config_update=lambda updated: _persist_upstream_auth_config(upstream, updated),
target_id=upstream.id,
target_name=upstream.name,
) as client:
client.login()
# 获取远端 Key 列表（支持自定义 managed_prefix）
backend/test_external_api_log.py
+677
View File
@@ -0,0 +1,677 @@
"""Tests for external API log persistence: logging, query, summary, and cleanup."""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from unittest.mock import patch
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from app.database import Base
from app.models.external_api_log import ExternalApiLog
@pytest.fixture
def engine():
e = create_engine("sqlite://", connect_args={"check_same_thread": False})
# Import all models so they register with Base.metadata
import app.models.admin_user # noqa: F401
import app.models.upstream # noqa: F401
import app.models.website # noqa: F401
import app.models.snapshot # noqa: F401
import app.models.webhook_config # noqa: F401
import app.models.notification_log # noqa: F401
import app.models.custom_page # noqa: F401
import app.models.revoked_token # noqa: F401
import app.models.upstream_key # noqa: F401
import app.models.external_api_log # noqa: F401
Base.metadata.create_all(bind=e)
yield e
Base.metadata.drop_all(bind=e)
@pytest.fixture
def session(engine):
TestingSession = sessionmaker(bind=engine)
db = TestingSession()
try:
yield db
finally:
db.close()
# ─────────────────────────────────────────────
# 1. Model: basic insert and read
# ─────────────────────────────────────────────
def test_insert_and_read(session):
now = datetime.now(timezone.utc)
log = ExternalApiLog(
direction="upstream",
target_type="upstream",
target_id=1,
target_name="test-upstream",
method="GET",
path="/api/v1/models",
url_host="api.test.local",
status_code=200,
success=True,
duration_ms=150,
created_at=now,
)
session.add(log)
session.commit()
rows = session.query(ExternalApiLog).all()
assert len(rows) == 1
assert rows[0].direction == "upstream"
assert rows[0].duration_ms == 150
assert rows[0].status_code == 200
assert rows[0].success is True
# ─────────────────────────────────────────────
# 2. Filters (direction, target_type, target_id, success, status_code, path)
# ─────────────────────────────────────────────
def _seed(session, count: int = 1, **overrides) -> list[ExternalApiLog]:
rows = []
for i in range(count):
defaults = dict(
direction="upstream",
target_type="upstream",
target_id=10 + i,
target_name=f"upstream-{i}",
method="GET",
path=f"/api/v1/endpoint/{i}",
url_host="api.test.local",
status_code=200,
success=True,
duration_ms=100 + i * 10,
created_at=datetime.now(timezone.utc),
)
defaults.update(overrides)
r = ExternalApiLog(**defaults)
session.add(r)
rows.append(r)
session.commit()
return rows
def test_filter_by_direction(session):
_seed(session, 1, direction="upstream")
_seed(session, 1, direction="website")
rows = session.query(ExternalApiLog).filter(ExternalApiLog.direction == "upstream").all()
assert len(rows) == 1
assert rows[0].direction == "upstream"
def test_filter_by_success(session):
_seed(session, 1, success=True)
_seed(session, 1, success=False, status_code=500, error_type="ServerError")
rows = session.query(ExternalApiLog).filter(ExternalApiLog.success == False).all() # noqa: E712
assert len(rows) == 1
assert rows[0].success is False
def test_filter_by_status_code(session):
_seed(session, 1, status_code=200)
_seed(session, 1, status_code=404, success=False, error_type="NotFound")
rows = session.query(ExternalApiLog).filter(ExternalApiLog.status_code == 404).all()
assert len(rows) == 1
assert rows[0].status_code == 404
def test_filter_by_target_id(session):
_seed(session, 1, target_id=42)
_seed(session, 1, target_id=99)
rows = session.query(ExternalApiLog).filter(ExternalApiLog.target_id == 42).all()
assert len(rows) == 1
assert rows[0].target_id == 42
def test_filter_by_path_like(session):
_seed(session, 1, path="/api/keys/list")
_seed(session, 1, path="/api/groups/list")
rows = session.query(ExternalApiLog).filter(ExternalApiLog.path.like("%keys%")).all()
assert len(rows) == 1
assert "keys" in rows[0].path
def test_pagination(session):
_seed(session, 5)
total = session.query(ExternalApiLog).count()
assert total == 5
page = session.query(ExternalApiLog).order_by(ExternalApiLog.id).offset(0).limit(2).all()
assert len(page) == 2
page2 = session.query(ExternalApiLog).order_by(ExternalApiLog.id).offset(2).limit(2).all()
assert len(page2) == 2
# ─────────────────────────────────────────────
# 3. Summary aggregation
# ─────────────────────────────────────────────
def test_summary_aggregation(session):
now = datetime.now(timezone.utc)
# 3 calls to same path: 2 success, 1 fail, various durations
for i in range(3):
session.add(ExternalApiLog(
direction="upstream",
target_type="upstream",
target_id=1,
target_name="agg-test",
method="GET",
path="/api/v1/status",
url_host="api.test.local",
status_code=200 if i < 2 else 500,
success=i < 2,
duration_ms=[100, 200, 300][i],
created_at=now,
))
# A different path
session.add(ExternalApiLog(
direction="website",
target_type="website",
target_id=2,
target_name="site-2",
method="POST",
path="/api/v1/accounts",
url_host="site.test.local",
status_code=201,
success=True,
duration_ms=500,
created_at=now,
))
session.commit()
from sqlalchemy import cast, case, func, Integer
slow_threshold = 250
cols = [
ExternalApiLog.direction,
ExternalApiLog.target_type,
ExternalApiLog.target_id,
ExternalApiLog.target_name,
ExternalApiLog.method,
ExternalApiLog.path,
func.count(ExternalApiLog.id).label("call_count"),
func.sum(cast(ExternalApiLog.success, Integer)).label("success_count"),
func.sum(cast(1 - cast(ExternalApiLog.success, Integer), Integer)).label("fail_count"),
func.avg(ExternalApiLog.duration_ms).label("avg_duration_ms"),
func.max(ExternalApiLog.duration_ms).label("max_duration_ms"),
func.sum(case((ExternalApiLog.duration_ms >= slow_threshold, 1), else_=0)).label("slow_count"),
]
rows = (
session.query(*cols)
.group_by(
ExternalApiLog.direction, ExternalApiLog.target_type,
ExternalApiLog.target_id, ExternalApiLog.target_name,
ExternalApiLog.method, ExternalApiLog.path,
)
.order_by(func.sum(ExternalApiLog.duration_ms).desc())
.all()
)
assert len(rows) == 2
# First row: upstream, 3 calls
r1 = rows[0]
assert r1.direction == "upstream"
assert r1.call_count == 3
assert r1.success_count == 2
assert r1.fail_count == 1
assert r1.slow_count == 1 # only 300ms >= 250
assert r1.max_duration_ms == 300
assert 190 <= r1.avg_duration_ms <= 210 # ~200
# Second row: website, 1 call
r2 = rows[1]
assert r2.direction == "website"
assert r2.call_count == 1
assert r2.success_count == 1
assert r2.fail_count == 0
# ─────────────────────────────────────────────
# 4. Cleanup (retention)
# ─────────────────────────────────────────────
def test_clean_expired_logs(session):
now = datetime.now(timezone.utc)
# Old record (40 days ago)
session.add(ExternalApiLog(
direction="upstream", target_type="upstream", method="GET",
path="/old", url_host="old.local",
status_code=200, success=True, duration_ms=10,
created_at=now - timedelta(days=40),
))
# Recent record (5 days ago)
session.add(ExternalApiLog(
direction="upstream", target_type="upstream", method="GET",
path="/recent", url_host="recent.local",
status_code=200, success=True, duration_ms=10,
created_at=now - timedelta(days=5),
))
session.commit()
assert session.query(ExternalApiLog).count() == 2
# Delete records older than 30 days
cutoff = now - timedelta(days=30)
deleted = session.query(ExternalApiLog).filter(
ExternalApiLog.created_at < cutoff
).delete(synchronize_session=False)
session.commit()
assert deleted == 1
assert session.query(ExternalApiLog).count() == 1
remaining = session.query(ExternalApiLog).first()
assert "/recent" in remaining.path
# ─────────────────────────────────────────────
# 5. log_external_api_call function
# ─────────────────────────────────────────────
def test_log_external_api_call_success(session):
"""Verify the logging helper writes the expected row."""
from app.services.external_api_logger import log_external_api_call
with patch("app.services.external_api_logger.SessionLocal", return_value=session):
log_external_api_call(
direction="upstream",
target_type="upstream",
method="POST",
path="/api/v1/test",
url_host="test.local",
duration_ms=250,
status_code=201,
success=True,
target_id=7,
target_name="gold-upstream",
)
rows = session.query(ExternalApiLog).all()
assert len(rows) == 1
r = rows[0]
assert r.direction == "upstream"
assert r.method == "POST"
assert r.path == "/api/v1/test"
assert r.url_host == "test.local"
assert r.duration_ms == 250
assert r.status_code == 201
assert r.success is True
assert r.target_id == 7
assert r.target_name == "gold-upstream"
assert r.error_type is None
assert r.error_message is None
def test_log_external_api_call_failure(session):
"""Failure path: error fields populated, success=False."""
from app.services.external_api_logger import log_external_api_call
with patch("app.services.external_api_logger.SessionLocal", return_value=session):
log_external_api_call(
direction="website",
target_type="website",
method="GET",
path="/api/v1/groups",
url_host="site.local",
duration_ms=5000,
status_code=502,
success=False,
target_id=3,
target_name="bad-site",
error_type="BadGateway",
error_message="upstream connection refused",
)
rows = session.query(ExternalApiLog).all()
assert len(rows) == 1
r = rows[0]
assert r.direction == "website"
assert r.success is False
assert r.status_code == 502
assert r.duration_ms == 5000
assert r.error_type == "BadGateway"
assert r.error_message == "upstream connection refused"
def test_log_no_sensitive_data(session):
"""Verify no request/response body or credentials are logged."""
from app.services.external_api_logger import log_external_api_call
with patch("app.services.external_api_logger.SessionLocal", return_value=session):
log_external_api_call(
direction="upstream",
target_type="upstream",
method="GET",
path="/api/v1/keys",
url_host="secure.local",
duration_ms=100,
status_code=200,
success=True,
)
rows = session.query(ExternalApiLog).all()
assert len(rows) == 1
r = rows[0]
# These fields MUST be empty — we never log bodies or credentials
assert not hasattr(r, "request_body")
assert not hasattr(r, "response_body")
assert not hasattr(r, "authorization")
assert not hasattr(r, "cookie")
assert not hasattr(r, "api_key")
# ─────────────────────────────────────────────
# 6. Time window on summary
# ─────────────────────────────────────────────
def test_summary_time_window(session):
now = datetime.now(timezone.utc)
# Recent (1 hour ago)
session.add(ExternalApiLog(
direction="upstream", target_type="upstream", method="GET",
path="/recent", url_host="h", status_code=200, success=True,
duration_ms=50, created_at=now - timedelta(hours=1),
))
# Old (48 hours ago)
session.add(ExternalApiLog(
direction="upstream", target_type="upstream", method="GET",
path="/old", url_host="h", status_code=200, success=True,
duration_ms=50, created_at=now - timedelta(hours=48),
))
session.commit()
# 24-hour window should only include the recent record
cutoff = now - timedelta(hours=24)
rows = session.query(ExternalApiLog).filter(ExternalApiLog.created_at >= cutoff).all()
assert len(rows) == 1
assert "/recent" in rows[0].path
# 72-hour window includes both
cutoff = now - timedelta(hours=72)
rows = session.query(ExternalApiLog).filter(ExternalApiLog.created_at >= cutoff).all()
assert len(rows) == 2
# ─────────────────────────────────────────────
# 7. UpstreamClient integration: 4xx/5xx logged as failure
# ─────────────────────────────────────────────
def _make_httpx_mock(status_code: int, body: dict | None = None):
"""Return a mock for httpx.Client.request that returns a fixed status.
Used via patch.object(client._client, 'request', ...) which replaces the
bound method, so the signature is (method, url, **kwargs) — no self.
"""
import httpx
def mock_request(method, url, **kwargs):
req = httpx.Request(method, url)
return httpx.Response(status_code, json=body or {}, request=req)
return mock_request
def test_upstream_client_500_logged_as_failure():
"""UpstreamClient._request() → HTTP 500 → log with success=False, status_code=500."""
from app.services.upstream_client import UpstreamClient
captured = {}
def capture(**kw):
captured.update(kw)
with patch("app.services.upstream_client.log_external_api_call", side_effect=capture):
client = UpstreamClient(
base_url="http://test.local",
api_prefix="",
auth_type="api_key",
auth_config={"key": "test", "header": "Authorization"},
target_id=1,
target_name="test-upstream",
)
with patch.object(client._client, "request", _make_httpx_mock(500)):
try:
client._request("GET", "/api/v1/models")
except Exception:
pass
assert captured.get("status_code") == 500
assert captured.get("success") is False
assert captured.get("error_type") == "HTTPStatus"
assert "500" in (captured.get("error_message") or "")
def test_upstream_client_429_logged_as_failure():
"""UpstreamClient._request() → HTTP 429 → log with success=False, status_code=429."""
from app.services.upstream_client import UpstreamClient
captured = {}
def capture(**kw):
captured.update(kw)
with patch("app.services.upstream_client.log_external_api_call", side_effect=capture):
client = UpstreamClient(
base_url="http://test.local",
api_prefix="",
auth_type="api_key",
auth_config={"key": "test", "header": "Authorization"},
target_id=1,
target_name="test-upstream",
)
with patch.object(client._client, "request", _make_httpx_mock(429)):
try:
client._request("GET", "/api/v1/rate-limited")
except Exception:
pass
assert captured.get("status_code") == 429
assert captured.get("success") is False
assert captured.get("error_type") == "HTTPStatus"
def test_upstream_client_200_logged_as_success():
"""UpstreamClient._request() → HTTP 200 → log with success=True, status_code=200."""
from app.services.upstream_client import UpstreamClient
captured = {}
def capture(**kw):
captured.update(kw)
with patch("app.services.upstream_client.log_external_api_call", side_effect=capture):
client = UpstreamClient(
base_url="http://test.local",
api_prefix="",
auth_type="api_key",
auth_config={"key": "test", "header": "Authorization"},
target_id=1,
target_name="test-upstream",
)
with patch.object(client._client, "request", _make_httpx_mock(200, {"data": []})):
client._request("GET", "/api/v1/groups")
assert captured.get("status_code") == 200
assert captured.get("success") is True
assert captured.get("error_type") is None
# ─────────────────────────────────────────────
# 8. Route-level tests — call handlers directly
# ─────────────────────────────────────────────
def test_route_list_empty(session):
"""list_external_api_logs with no data returns []."""
from app.routers.external_api_logs import list_external_api_logs
result = list_external_api_logs(
direction=None, target_type=None, target_id=None,
success=None, status_code=None, path=None,
limit=100, offset=0, db=session, _=None,
)
assert result == []
def test_route_list_with_filters(session):
"""list_external_api_logs with filters returns filtered results."""
from app.routers.external_api_logs import list_external_api_logs
now = datetime.now(timezone.utc)
for i in range(3):
session.add(ExternalApiLog(
direction="upstream" if i < 2 else "website",
target_type="upstream" if i < 2 else "website",
target_id=i + 1, target_name=f"tgt-{i}",
method="GET", path=f"/path/{i}", url_host="h.local",
status_code=200 if i < 2 else 500, success=i < 2,
duration_ms=100 + i, created_at=now,
))
session.commit()
_empty = None # sentinel for "no filter"
# Filter by direction
result = list_external_api_logs(
direction="website", target_type=_empty, target_id=_empty,
success=_empty, status_code=_empty, path=_empty,
limit=100, offset=0, db=session, _=None,
)
assert len(result) == 1 and result[0].direction == "website"
# Filter by success
result = list_external_api_logs(
direction=_empty, target_type=_empty, target_id=_empty,
success=False, status_code=_empty, path=_empty,
limit=100, offset=0, db=session, _=None,
)
assert len(result) == 1 and result[0].success is False
# Filter by status_code
result = list_external_api_logs(
direction=_empty, target_type=_empty, target_id=_empty,
success=_empty, status_code=500, path=_empty,
limit=100, offset=0, db=session, _=None,
)
assert len(result) == 1 and result[0].status_code == 500
# Filter by path
result = list_external_api_logs(
direction=_empty, target_type=_empty, target_id=_empty,
success=_empty, status_code=_empty, path="/path/0",
limit=100, offset=0, db=session, _=None,
)
assert len(result) == 1 and "/path/0" in result[0].path
# Filter by target_id
result = list_external_api_logs(
direction=_empty, target_type=_empty, target_id=2,
success=_empty, status_code=_empty, path=_empty,
limit=100, offset=0, db=session, _=None,
)
assert len(result) == 1 and result[0].target_id == 2
def test_route_list_pagination(session):
"""list_external_api_logs respects limit/offset."""
from app.routers.external_api_logs import list_external_api_logs
now = datetime.now(timezone.utc)
for i in range(10):
session.add(ExternalApiLog(
direction="upstream", target_type="upstream",
method="GET", path=f"/p/{i}", url_host="h",
status_code=200, success=True, duration_ms=i, created_at=now,
))
session.commit()
_e = None
assert len(list_external_api_logs(
direction=_e, target_type=_e, target_id=_e,
success=_e, status_code=_e, path=_e,
limit=3, offset=0, db=session, _=None,
)) == 3
assert len(list_external_api_logs(
direction=_e, target_type=_e, target_id=_e,
success=_e, status_code=_e, path=_e,
limit=3, offset=3, db=session, _=None,
)) == 3
assert len(list_external_api_logs(
direction=_e, target_type=_e, target_id=_e,
success=_e, status_code=_e, path=_e,
limit=3, offset=20, db=session, _=None,
)) == 0
def test_route_summary_hours_window(session):
"""get_external_api_log_summary respects hours parameter."""
from app.routers.external_api_logs import get_external_api_log_summary
now = datetime.now(timezone.utc)
# Recent (30 minutes ago — well within 1-hour window)
session.add(ExternalApiLog(direction="upstream", target_type="upstream",
method="GET", path="/recent", url_host="h",
status_code=200, success=True, duration_ms=50,
created_at=now - timedelta(minutes=30)))
# Old (48 hours ago — outside 1-hour window)
session.add(ExternalApiLog(direction="upstream", target_type="upstream",
method="GET", path="/old", url_host="h",
status_code=200, success=True, duration_ms=50,
created_at=now - timedelta(hours=48)))
session.commit()
# hours=1 → only recent
result = get_external_api_log_summary(
slow_ms=None, hours=1, limit=50, db=session, _=None,
)
assert len(result) == 1 and result[0].path == "/recent"
# hours=72 → both
result = get_external_api_log_summary(
slow_ms=None, hours=72, limit=50, db=session, _=None,
)
assert len(result) == 2
def test_route_summary_limit(session):
"""get_external_api_log_summary respects limit parameter."""
from app.routers.external_api_logs import get_external_api_log_summary
now = datetime.now(timezone.utc)
for i in range(10):
session.add(ExternalApiLog(direction="upstream", target_type="upstream",
method="GET", path=f"/path-{i}", url_host="h",
status_code=200, success=True, duration_ms=50 + i, created_at=now))
session.commit()
result = get_external_api_log_summary(
slow_ms=None, hours=24, limit=3, db=session, _=None,
)
assert len(result) == 3
def test_route_summary_slow_threshold(session):
"""get_external_api_log_summary respects slow_ms parameter."""
from app.routers.external_api_logs import get_external_api_log_summary
now = datetime.now(timezone.utc)
session.add(ExternalApiLog(direction="upstream", target_type="upstream",
method="GET", path="/fast", url_host="h",
status_code=200, success=True, duration_ms=100, created_at=now))
session.add(ExternalApiLog(direction="upstream", target_type="upstream",
method="GET", path="/fast2", url_host="h",
status_code=200, success=True, duration_ms=200, created_at=now))
session.add(ExternalApiLog(direction="upstream", target_type="upstream",
method="GET", path="/slow", url_host="h",
status_code=200, success=True, duration_ms=5000, created_at=now))
session.commit()
result = get_external_api_log_summary(
slow_ms=1000, hours=24, limit=50, db=session, _=None,
)
slow_paths = [r for r in result if r.slow_count > 0]
assert len(slow_paths) == 1
assert slow_paths[0].path == "/slow"