feat: 上游 Key 唯一化、分组导入跳过、账号导入平台识别&远端校验&base_url 注入

- 上游 Key 命名改为 {prefix}-{upstream.id}-{safe_group_name}-{group_id} - 唯一约束 (upstream_id, group_id, managed_prefix) 加 managed_prefix 列 - 上游检测成功时同步 Key 状态，远端已删/分组已删自动清理 - 重复分组导入跳过，目标网站已存在同名分组返回 exists - 账号导入平台自动识别（auto/manual 模式） - 全选可导入 Key 按钮 + 目标分组自动匹配 - 导入幂等：已导入过的 Key 校验远端账号，不存在则重建 - 新增同步接口 POST /sync-imported-upstream-keys - account_exists() 通过拉取账号列表判断，避免 404 误判 - credentials.base_url 注入来源上游地址，避免 401 - 前端导入弹窗自动同步+刷新按钮+并发/优先级设置 - 新增 12 个测试覆盖同步、幂等、远端删除、校验失败路径
2026-05-21 01:16:39 +08:00
parent 0a27bba296
commit 6044b00685
18 changed files with 3112 additions and 50 deletions
@@ -11,6 +11,7 @@ from sqlalchemy.orm import Session

 from app.database import SessionLocal
 from app.models.upstream import Upstream
+from app.models.upstream_key import UpstreamGeneratedKey
 from app.models.snapshot import UpstreamRateSnapshot
 from app.services.upstream_client import UpstreamClient, UpstreamError, build_snapshot
 from app.services.snapshot_service import diff_snapshots, prune_snapshots
@@ -130,7 +131,20 @@ def _check_upstream(upstream_id: int) -> None:
    finally:
        db.close()

-    # ── Phase 2: notifications (independent sessions) ──────────────
+    # ── Phase 2: key sync (independent session) ───────────────────
+    if snapshot:
+        captured_at = snapshot.get("captured_at")
+        if isinstance(captured_at, str):
+            from datetime import datetime as dt
+            try:
+                captured_at = dt.fromisoformat(captured_at)
+            except Exception:
+                captured_at = datetime.now(timezone.utc)
+        elif captured_at is None:
+            captured_at = datetime.now(timezone.utc)
+        _sync_upstream_keys(upstream_id, snapshot, captured_at)
+
+    # ── Phase 3: notifications (independent sessions) ──────────────
    if was_unhealthy:
        _notify_status(upstream_id, upstream.name, upstream.base_url, "upstream_recovered")

@@ -170,6 +184,63 @@ def _notify_rate_changed(
        db.close()


+def _sync_upstream_keys(upstream_id: int, snapshot: dict[str, Any], captured_at: datetime) -> None:
+    """上游检测成功后同步 SmartUp Key 状态（远端删除/分组删除）。"""
+    db = SessionLocal()
+    try:
+        active_group_ids = set(snapshot.get("groups", {}).keys())
+        key_rows = (
+            db.query(UpstreamGeneratedKey)
+            .filter(
+                UpstreamGeneratedKey.upstream_id == upstream_id,
+                UpstreamGeneratedKey.key_name.like("SmartUp-%"),
+            )
+            .all()
+        )
+        auth_config = json.loads(
+            db.query(Upstream).filter(Upstream.id == upstream_id).first().auth_config_json or "{}"
+        )
+        # 用 UpstreamClient 查询远端活跃 Key ID 集合
+        remote_key_ids: set[str] | None = None  # None=查询失败，set()=查询成功但为空
+        try:
+            upstream = db.query(Upstream).filter(Upstream.id == upstream_id).first()
+            if upstream:
+                with UpstreamClient(
+                    base_url=upstream.base_url,
+                    api_prefix=upstream.api_prefix,
+                    auth_type=upstream.auth_type,
+                    auth_config=auth_config,
+                    timeout=float(upstream.timeout_seconds),
+                ) as client:
+                    client.login()
+                    remote_keys = client.list_api_keys(search="SmartUp", status="active")
+                    remote_key_ids = {
+                        str(k["id"]) for k in remote_keys if k.get("id")
+                    }
+        except Exception as exc:
+            logger.warning("sync upstream keys list failed for %s: %s", upstream_id, exc)
+
+        for row in key_rows:
+            # 1. 分组已不在当前快照中 → 删除本地记录
+            if row.group_id not in active_group_ids:
+                db.delete(row)
+                logger.info("removed key %s (group %s no longer in snapshot)", row.id, row.group_id)
+                continue
+            # 2. 远端查询成功但 key_id 不在列表中 → 删除本地记录
+            if row.key_id and remote_key_ids is not None and row.key_id not in remote_key_ids:
+                db.delete(row)
+                logger.info("removed key %s (key_id %s gone from remote)", row.id, row.key_id)
+                continue
+            # 3. 更新同步时间戳（仅当查询成功且 Key 仍在远端时）
+            if remote_key_ids is not None and row.key_id in remote_key_ids:
+                row.updated_at = captured_at
+        db.commit()
+    except Exception:
+        logger.exception("key sync failed for upstream %s", upstream_id)
+    finally:
+        db.close()
+
+
 def _sync_website_bindings(upstream_id: int, changes: list[dict[str, Any]]) -> None:
    db = SessionLocal()
    try:
@@ -62,6 +62,49 @@ def _find_user_id(value: Any) -> str:
    return ""


+def mask_secret(value: Any) -> str:
+    text = str(value or "")
+    if not text:
+        return ""
+    if len(text) <= 8:
+        return text[:2] + "****" + text[-2:] if len(text) > 4 else "****"
+    return text[:4] + "**********" + text[-4:]
+
+
+def _unwrap_data(value: Any) -> Any:
+    if isinstance(value, dict) and "data" in value and ("code" in value or "message" in value):
+        return value.get("data")
+    return value
+
+
+def _extract_id(value: Any) -> str:
+    if isinstance(value, dict):
+        for key in ("id", "key_id", "keyId"):
+            candidate = value.get(key)
+            if candidate is not None:
+                return str(candidate)
+        for key in ("data", "result", "key", "api_key"):
+            found = _extract_id(value.get(key))
+            if found:
+                return found
+    return ""
+
+
+def _extract_key_value(value: Any) -> str:
+    if isinstance(value, str):
+        return value
+    if isinstance(value, dict):
+        for key in ("key", "api_key", "apiKey", "token", "value"):
+            candidate = value.get(key)
+            if isinstance(candidate, str) and candidate:
+                return candidate
+        for key in ("data", "result", "api_key", "key"):
+            found = _extract_key_value(value.get(key))
+            if found:
+                return found
+    return ""
+
+
 def _unwrap_list(value: Any) -> Optional[list[dict[str, Any]]]:
    def _normalize(lst: list) -> list[dict[str, Any]]:
        out = []
@@ -360,3 +403,107 @@ class UpstreamClient:
            return float(value)
        except (ValueError, TypeError):
            return None
+
+    def list_api_keys(
+        self,
+        search: str = "",
+        group_id: str | int | None = None,
+        status: str = "active",
+        endpoint: str = "/keys",
+    ) -> list[dict[str, Any]]:
+        """查询远端上游 Key 列表，支持按名称搜索、分组筛选、状态筛选。"""
+        params: dict[str, Any] = {}
+        if search:
+            params["search"] = search
+        if group_id is not None:
+            params["group_id"] = int(group_id) if str(group_id).isdigit() else group_id
+        if status:
+            params["status"] = status
+        url = self._url(endpoint)
+        resp = self._client.request(
+            "GET",
+            url,
+            params=params if params else None,
+            headers=self._headers(),
+            cookies=self._cookies,
+        )
+        resp.raise_for_status()
+        data = resp.json()
+        if isinstance(data, list):
+            return data
+        if isinstance(data, dict):
+            # 尝试展开常见的包装结构
+            for top_key in ("data", "result", "response"):
+                val = data.get(top_key)
+                if isinstance(val, list):
+                    return val
+                if isinstance(val, dict):
+                    for inner_key in ("items", "keys", "list", "records", "data"):
+                        inner = val.get(inner_key)
+                        if isinstance(inner, list):
+                            return inner
+            # 顶层本身就是 list-like wrapper
+            for key in ("items", "keys", "list", "records"):
+                val = data.get(key)
+                if isinstance(val, list):
+                    return val
+        raise UpstreamError(f"unexpected keys response type: {type(data).__name__}")
+
+    def delete_api_key(self, key_id: str, endpoint: str = "/keys") -> None:
+        """删除远端上游上的一个 Key。"""
+        self._request("DELETE", f"{endpoint}/{key_id}")
+
+    def find_smartup_group_key(
+        self,
+        group_id: str | int,
+        expected_name: str,
+        prefix: str = "SmartUp",
+    ) -> dict[str, Any] | None:
+        """查找同一上游分组下是否已存在 SmartUp 前缀的 Key。
+
+        匹配规则：key_name 等于 expected_name，且以 prefix 开头。
+        返回匹配到的第一个 Key，或 None。
+        """
+        gid = int(group_id) if str(group_id).isdigit() else group_id
+        keys = self.list_api_keys(search=prefix, group_id=gid, status="active")
+        for k in keys:
+            name = k.get("name") or k.get("key_name") or ""
+            if name == expected_name:
+                return k
+            # 部分后端返回的 name 可能带空格或 trimming
+            if name.strip() == expected_name.strip():
+                return k
+        return None
+
+    def create_api_key(
+        self,
+        name: str,
+        group_id: str | int,
+        quota: float = 0,
+        expires_in_days: int | None = None,
+        rate_limit_5h: float = 0,
+        rate_limit_1d: float = 0,
+        rate_limit_7d: float = 0,
+        endpoint: str = "/keys",
+    ) -> dict[str, Any]:
+        body: dict[str, Any] = {
+            "name": name,
+            "group_id": int(group_id) if str(group_id).isdigit() else group_id,
+            "quota": quota,
+            "rate_limit_5h": rate_limit_5h,
+            "rate_limit_1d": rate_limit_1d,
+            "rate_limit_7d": rate_limit_7d,
+        }
+        if expires_in_days:
+            body["expires_in_days"] = expires_in_days
+        resp = self._request("POST", endpoint, body)
+        data = _unwrap_data(resp)
+        key_value = _extract_key_value(data)
+        if not key_value:
+            raise UpstreamError("key create response did not include key")
+        return {
+            "id": _extract_id(data),
+            "key": key_value,
+            "masked_key": mask_secret(key_value),
+            "raw": data if isinstance(data, dict) else {"value": data},
+        }
@@ -1,5 +1,6 @@
 from __future__ import annotations

+import logging
 from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
 from typing import Any
 from urllib.parse import quote
@@ -8,11 +9,39 @@ import httpx

 from app.utils.number import decimal_string

+logger = logging.getLogger(__name__)
+

 class WebsiteError(RuntimeError):
    pass


+def _friendly_http_error(exc: httpx.HTTPStatusError) -> str:
+    """将常见 HTTP 错误转换为中文友好提示，原始信息保留在日志中。"""
+    status = exc.response.status_code
+    url = exc.request.url if exc.request else "?"
+    logger.warning("website_client HTTP %s from %s: %s", status, url, exc)
+    if status == 401:
+        return "目标网站认证失败，请检查 Admin API Key / JWT 是否正确"
+    if status == 403:
+        return "目标网站权限不足，请检查当前凭证是否有分组管理权限"
+    if status == 404:
+        return f"目标网站接口不存在，请检查 API Prefix 和分组接口路径（{exc.response.url.path}）"
+    if 500 <= status < 600:
+        return "目标网站服务异常，请稍后重试"
+    return f"目标网站返回错误（HTTP {status}）"
+
+
+def _friendly_connection_error(exc: Exception) -> str:
+    """将网络/超时异常转换为中文友好提示。"""
+    logger.warning("website_client connection error: %s", exc)
+    if isinstance(exc, httpx.TimeoutException):
+        return "目标网站请求超时，请检查网络连接和 API 地址是否正确"
+    if isinstance(exc, httpx.ConnectError):
+        return "无法连接目标网站，请检查 API 地址和网络连通性"
+    return f"目标网站通信异常：{exc}"
+
+
 def parse_positive_decimal(value: Any) -> Decimal | None:
    if value is None or value == "":
        return None
@@ -59,6 +88,19 @@ def _unwrap_data(value: Any) -> Any:
    return value


+def _extract_id(value: Any) -> str:
+    if isinstance(value, dict):
+        for key in ("id", "account_id", "accountId", "group_id", "groupId"):
+            candidate = value.get(key)
+            if candidate is not None:
+                return str(candidate)
+        for key in ("data", "result", "account", "group"):
+            found = _extract_id(value.get(key))
+            if found:
+                return found
+    return ""
+
+
 def normalize_groups(value: Any) -> list[dict[str, Any]]:
    raw = _unwrap_data(value)
    if isinstance(raw, dict):
@@ -129,24 +171,111 @@ class Sub2ApiWebsiteClient:
        return headers

    def _request(self, method: str, path: str, body: Any = None) -> Any:
-        resp = self._client.request(method, self._url(path), json=body, headers=self._headers())
-        resp.raise_for_status()
+        try:
+            resp = self._client.request(method, self._url(path), json=body, headers=self._headers())
+            resp.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            raise WebsiteError(_friendly_http_error(exc)) from exc
+        except httpx.TimeoutException as exc:
+            raise WebsiteError(_friendly_connection_error(exc)) from exc
+        except httpx.ConnectError as exc:
+            raise WebsiteError(_friendly_connection_error(exc)) from exc
        if not resp.content:
            return None
        text = resp.text
        if "application/json" not in resp.headers.get("content-type", "") and text.lstrip().startswith("<"):
-            raise WebsiteError(f"{method} {path} returned HTML, not JSON")
+            raise WebsiteError(f"{method} {path} 返回了 HTML，请检查接口地址是否正确")
        return resp.json()

    def get_groups(self, endpoint: str = "/groups") -> list[dict[str, Any]]:
-        errors: list[str] = []
+        """拉取分组列表，尝试 endpoint 和 fallback /groups/all。"""
+        last_error: Exception | None = None
+        tried_paths: list[str] = []
        for path in [endpoint, "/groups/all"]:
+            tried_paths.append(path)
            try:
                return normalize_groups(self._request("GET", path))
+            except WebsiteError as exc:
+                msg = str(exc)
+                # 认证/权限类错误：直接抛出，不需要尝试 fallback
+                if "认证失败" in msg or "权限不足" in msg:
+                    raise
+                # 404/5xx 等路径相关错误，试试另一个路径
+                last_error = exc
            except Exception as exc:
-                errors.append(f"{path}: {exc}")
-        raise WebsiteError("; ".join(errors))
+                last_error = exc
+                logger.info("get_groups fallback %s failed: %s", path, exc)
+
+        msg = str(last_error) if last_error else "拉取分组失败"
+        raise WebsiteError(f"{msg}（尝试接口：{'、'.join(tried_paths)}）")

    def update_group_rate(self, endpoint_template: str, group_id: str, rate: Decimal) -> Any:
        path = endpoint_template.replace("{id}", quote(group_id, safe=""))
        return self._request("PUT", path, {"rate_multiplier": float(rate)})
+
+    def create_group(self, body: dict[str, Any], endpoint: str = "/groups") -> dict[str, Any]:
+        resp = self._request("POST", endpoint, body)
+        data = _unwrap_data(resp)
+        return data if isinstance(data, dict) else {"value": data}
+
+    def create_account(self, body: dict[str, Any], endpoint: str = "/accounts") -> dict[str, Any]:
+        resp = self._request("POST", endpoint, body)
+        data = _unwrap_data(resp)
+        return data if isinstance(data, dict) else {"value": data}
+
+    @staticmethod
+    def _unwrap_list(value: dict) -> list | None:
+        """递归展开嵌套的列表包装：data.items、data.data、items、accounts 等。"""
+        if isinstance(value, list):
+            return value
+        if not isinstance(value, dict):
+            return None
+        # 先看顶层
+        for key in ("items", "accounts", "records", "list", "data"):
+            v = value.get(key)
+            if isinstance(v, list):
+                return v
+        # 再看 data.items、data.records、data.list 等嵌套
+        data_val = value.get("data")
+        if isinstance(data_val, dict):
+            for key in ("items", "records", "list", "data", "accounts"):
+                v = data_val.get(key)
+                if isinstance(v, list):
+                    return v
+        return None
+
+    def _get_account_ids(self, endpoint: str = "/accounts") -> set[str] | None:
+        """拉取远端账号列表。成功返回 ID 集合（可能为空），解析失败返回 None。"""
+        try:
+            resp = self._request("GET", endpoint)
+        except Exception:
+            logger.warning("account list fetch failed for %s", endpoint, exc_info=True)
+            return None
+        items = self._unwrap_list(resp)
+        if items is None:
+            logger.warning("account list unexpected format for %s", endpoint)
+            return None
+        ids: set[str] = set()
+        for item in items:
+            item_id = self.extract_id(item)
+            if item_id:
+                ids.add(item_id)
+        return ids
+
+    def account_exists(self, account_id: str, endpoint: str = "/accounts") -> bool | None:
+        """检查目标账号是否存在。
+        
+        优先拉取账号列表判断：
+        - 列表成功取到 → return account_id in ids（True=存在，False=已删除）
+        - 列表取不到（None）→ return None（校验失败，不清本地）
+        返回 True=存在，False=已删除，None=校验失败。
+        """
+        ids = self._get_account_ids(endpoint)
+        if ids is None:
+            logger.warning("account_exists cannot verify %s: list fetch failed", account_id)
+            return None
+        return account_id in ids
+
+    @staticmethod
+    def extract_id(value: Any) -> str:
+        return _extract_id(value)