Initial commit
This commit is contained in:
liumangmang
@@ -0,0 +1 @@
|
||||
# services package
|
||||
@@ -0,0 +1,156 @@
|
||||
"""APScheduler background scheduler for upstream checks."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import SessionLocal
|
||||
from app.models.upstream import Upstream
|
||||
from app.models.snapshot import UpstreamRateSnapshot
|
||||
from app.services.upstream_client import UpstreamClient, UpstreamError, build_snapshot
|
||||
from app.services.snapshot_service import diff_snapshots
|
||||
from app.services import webhook_service
|
||||
from app.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_scheduler = BackgroundScheduler(timezone="UTC")
|
||||
|
||||
|
||||
def get_scheduler() -> BackgroundScheduler:
|
||||
return _scheduler
|
||||
|
||||
|
||||
def _check_upstream(upstream_id: int) -> None:
|
||||
"""Full upstream check executed by scheduler (runs in thread)."""
|
||||
settings = get_settings()
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
upstream = db.query(Upstream).filter(Upstream.id == upstream_id).first()
|
||||
if not upstream or not upstream.enabled:
|
||||
_remove_job(upstream_id)
|
||||
return
|
||||
|
||||
auth_config = json.loads(upstream.auth_config_json or "{}")
|
||||
client = UpstreamClient(
|
||||
base_url=upstream.base_url,
|
||||
api_prefix=upstream.api_prefix,
|
||||
auth_type=upstream.auth_type,
|
||||
auth_config=auth_config,
|
||||
timeout=float(upstream.timeout_seconds),
|
||||
)
|
||||
|
||||
was_unhealthy = upstream.last_status == "unhealthy"
|
||||
|
||||
try:
|
||||
client.login()
|
||||
groups = client.get_available_groups(upstream.groups_endpoint)
|
||||
raw_rates = client.get_group_rates(upstream.rate_endpoint)
|
||||
snapshot = build_snapshot(
|
||||
upstream.id, upstream.base_url, upstream.api_prefix, groups, raw_rates
|
||||
)
|
||||
except Exception as exc:
|
||||
# failure path
|
||||
upstream.consecutive_failures = (upstream.consecutive_failures or 0) + 1
|
||||
upstream.last_error = str(exc)
|
||||
upstream.last_checked_at = datetime.now(timezone.utc)
|
||||
threshold = settings.unhealthy_threshold
|
||||
if upstream.consecutive_failures >= threshold and upstream.last_status != "unhealthy":
|
||||
upstream.last_status = "unhealthy"
|
||||
db.commit()
|
||||
webhook_service.send_status_event(
|
||||
db, upstream.id, upstream.name, upstream.base_url,
|
||||
"upstream_unhealthy", str(exc)
|
||||
)
|
||||
else:
|
||||
db.commit()
|
||||
logger.warning("upstream %s check failed: %s", upstream.name, exc)
|
||||
return
|
||||
|
||||
# success path
|
||||
prev_snapshot_row = (
|
||||
db.query(UpstreamRateSnapshot)
|
||||
.filter(UpstreamRateSnapshot.upstream_id == upstream_id)
|
||||
.order_by(UpstreamRateSnapshot.captured_at.desc())
|
||||
.first()
|
||||
)
|
||||
previous = json.loads(prev_snapshot_row.snapshot_json) if prev_snapshot_row else None
|
||||
changes = diff_snapshots(previous, snapshot)
|
||||
|
||||
# save new snapshot
|
||||
new_row = UpstreamRateSnapshot(
|
||||
upstream_id=upstream_id,
|
||||
snapshot_json=json.dumps(snapshot, ensure_ascii=False),
|
||||
captured_at=datetime.now(timezone.utc),
|
||||
)
|
||||
db.add(new_row)
|
||||
|
||||
# update upstream status
|
||||
upstream.last_status = "healthy"
|
||||
upstream.last_checked_at = datetime.now(timezone.utc)
|
||||
upstream.last_error = None
|
||||
upstream.consecutive_failures = 0
|
||||
db.commit()
|
||||
|
||||
if was_unhealthy:
|
||||
webhook_service.send_status_event(
|
||||
db, upstream.id, upstream.name, upstream.base_url, "upstream_recovered"
|
||||
)
|
||||
|
||||
if changes:
|
||||
webhook_service.send_rate_changed(
|
||||
db, upstream.id, upstream.name, upstream.base_url, changes
|
||||
)
|
||||
logger.info("upstream %s: %d rate change(s)", upstream.name, len(changes))
|
||||
else:
|
||||
logger.debug("upstream %s: no changes", upstream.name)
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _remove_job(upstream_id: int) -> None:
|
||||
job_id = f"upstream_{upstream_id}"
|
||||
if _scheduler.get_job(job_id):
|
||||
_scheduler.remove_job(job_id)
|
||||
|
||||
|
||||
def refresh_upstream(upstream_id: int, interval_seconds: int = 0, enabled: bool = True) -> None:
|
||||
"""Add/update/remove a scheduler job for the given upstream."""
|
||||
job_id = f"upstream_{upstream_id}"
|
||||
if not enabled or interval_seconds <= 0:
|
||||
_remove_job(upstream_id)
|
||||
return
|
||||
_scheduler.add_job(
|
||||
_check_upstream,
|
||||
"interval",
|
||||
seconds=interval_seconds,
|
||||
id=job_id,
|
||||
args=[upstream_id],
|
||||
replace_existing=True,
|
||||
coalesce=True,
|
||||
max_instances=1,
|
||||
)
|
||||
logger.info("scheduler job %s set to %ds interval", job_id, interval_seconds)
|
||||
|
||||
|
||||
def start_scheduler() -> None:
|
||||
"""Start scheduler and load all enabled upstreams."""
|
||||
_scheduler.start()
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
upstreams = db.query(Upstream).filter(Upstream.enabled == True).all()
|
||||
for u in upstreams:
|
||||
refresh_upstream(u.id, u.check_interval_seconds, u.enabled)
|
||||
logger.info("scheduler started with %d upstream job(s)", len(upstreams))
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def stop_scheduler() -> None:
|
||||
if _scheduler.running:
|
||||
_scheduler.shutdown(wait=False)
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Snapshot diff logic."""
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
def diff_snapshots(
|
||||
previous: Optional[dict[str, Any]],
|
||||
current: dict[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return list of rate changes between previous and current snapshots.
|
||||
Returns empty list if previous is None (first check)."""
|
||||
if not previous:
|
||||
return []
|
||||
old_groups: dict[str, Any] = previous.get("groups") or {}
|
||||
new_groups: dict[str, Any] = current.get("groups") or {}
|
||||
changes: list[dict[str, Any]] = []
|
||||
for gid, new_g in sorted(new_groups.items()):
|
||||
if not isinstance(new_g, dict):
|
||||
continue
|
||||
old_g = old_groups.get(gid)
|
||||
old_rate = old_g.get("rate") if isinstance(old_g, dict) else None
|
||||
new_rate = new_g.get("rate")
|
||||
if old_rate != new_rate:
|
||||
changes.append({
|
||||
"group_id": gid,
|
||||
"group_name": new_g.get("group_name", ""),
|
||||
"platform": new_g.get("platform", ""),
|
||||
"old_rate": old_rate,
|
||||
"new_rate": new_rate,
|
||||
})
|
||||
for gid, old_g in sorted(old_groups.items()):
|
||||
if gid not in new_groups and isinstance(old_g, dict):
|
||||
changes.append({
|
||||
"group_id": gid,
|
||||
"group_name": old_g.get("group_name", ""),
|
||||
"platform": old_g.get("platform", ""),
|
||||
"old_rate": old_g.get("rate"),
|
||||
"new_rate": None,
|
||||
})
|
||||
return changes
|
||||
@@ -0,0 +1,217 @@
|
||||
"""Upstream HTTP client — ported from monitor_ai98pro_group_rates.py."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import urljoin
|
||||
import httpx
|
||||
|
||||
|
||||
class UpstreamError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _find_token(value: Any) -> str:
|
||||
if isinstance(value, str) and value.count(".") >= 2:
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
for key in ("token", "access_token", "accessToken", "jwt", "auth_token", "authToken"):
|
||||
candidate = value.get(key)
|
||||
if isinstance(candidate, str) and candidate:
|
||||
return candidate
|
||||
for key in ("data", "result", "user", "session"):
|
||||
tok = _find_token(value.get(key))
|
||||
if tok:
|
||||
return tok
|
||||
return ""
|
||||
|
||||
|
||||
def _unwrap_list(value: Any) -> Optional[list[dict[str, Any]]]:
|
||||
if isinstance(value, list):
|
||||
return [i for i in value if isinstance(i, dict)]
|
||||
if isinstance(value, dict):
|
||||
for key in ("data", "items", "groups", "available_groups", "availableGroups"):
|
||||
nested = value.get(key)
|
||||
if isinstance(nested, list):
|
||||
return [i for i in nested if isinstance(i, dict)]
|
||||
return None
|
||||
|
||||
|
||||
def _decimal_str(value: Any) -> str:
|
||||
if value is None or value == "":
|
||||
return ""
|
||||
try:
|
||||
d = Decimal(str(value))
|
||||
except (InvalidOperation, ValueError):
|
||||
return str(value)
|
||||
n = d.normalize()
|
||||
if n == n.to_integral():
|
||||
return str(n.quantize(Decimal("1")))
|
||||
return format(n, "f")
|
||||
|
||||
|
||||
def _group_id(group: dict[str, Any]) -> str:
|
||||
for key in ("id", "group_id", "groupId"):
|
||||
v = group.get(key)
|
||||
if v is not None:
|
||||
return str(v)
|
||||
name = str(group.get("name") or group.get("group_name") or "")
|
||||
platform = str(group.get("platform") or "")
|
||||
return f"{platform}:{name}"
|
||||
|
||||
|
||||
def _rate_from_group(group: dict[str, Any]) -> str:
|
||||
for key in (
|
||||
"user_rate_multiplier", "userRateMultiplier",
|
||||
"effective_rate_multiplier", "effectiveRateMultiplier",
|
||||
"rate_multiplier", "rateMultiplier",
|
||||
):
|
||||
r = _decimal_str(group.get(key))
|
||||
if r:
|
||||
return r
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_rates_map(raw: Any) -> dict[str, str]:
|
||||
if raw is None:
|
||||
return {}
|
||||
if isinstance(raw, dict):
|
||||
candidates = raw
|
||||
for key in ("data", "rates", "group_rates", "groupRates"):
|
||||
nested = raw.get(key)
|
||||
if isinstance(nested, dict):
|
||||
candidates = nested
|
||||
break
|
||||
result: dict[str, str] = {}
|
||||
for k, v in candidates.items():
|
||||
if isinstance(v, dict):
|
||||
r = _decimal_str(
|
||||
v.get("rate_multiplier") or v.get("rateMultiplier")
|
||||
or v.get("user_rate_multiplier") or v.get("userRateMultiplier")
|
||||
)
|
||||
else:
|
||||
r = _decimal_str(v)
|
||||
if r:
|
||||
result[str(k)] = r
|
||||
return result
|
||||
if isinstance(raw, list):
|
||||
result = {}
|
||||
for item in raw:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
gid = _group_id(item)
|
||||
rate = _rate_from_group(item)
|
||||
if gid and rate:
|
||||
result[gid] = rate
|
||||
return result
|
||||
return {}
|
||||
|
||||
|
||||
def build_snapshot(upstream_id: int, base_url: str, api_prefix: str,
|
||||
groups: list[dict[str, Any]], raw_rates: Any) -> dict[str, Any]:
|
||||
from datetime import datetime, timezone
|
||||
override_rates = _extract_rates_map(raw_rates)
|
||||
entries: dict[str, dict[str, Any]] = {}
|
||||
for g in groups:
|
||||
gid = _group_id(g)
|
||||
default_rate = _rate_from_group(g)
|
||||
effective_rate = override_rates.get(gid, default_rate)
|
||||
entries[gid] = {
|
||||
"group_id": gid,
|
||||
"group_name": g.get("name") or g.get("group_name") or "",
|
||||
"platform": g.get("platform") or "",
|
||||
"rate": effective_rate,
|
||||
"default_rate": default_rate,
|
||||
"override_rate": override_rates.get(gid, ""),
|
||||
}
|
||||
return {
|
||||
"upstream_id": upstream_id,
|
||||
"base_url": base_url.rstrip("/"),
|
||||
"api_prefix": api_prefix,
|
||||
"captured_at": datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds"),
|
||||
"groups": entries,
|
||||
}
|
||||
|
||||
|
||||
class UpstreamClient:
|
||||
"""Sync HTTP client that handles all auth types."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
api_prefix: str,
|
||||
auth_type: str,
|
||||
auth_config: dict[str, Any],
|
||||
timeout: float = 30.0,
|
||||
) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.api_prefix = api_prefix.strip("/")
|
||||
self.auth_type = auth_type
|
||||
self.auth_config = auth_config
|
||||
self.timeout = timeout
|
||||
self._token: str = ""
|
||||
|
||||
def _url(self, path: str) -> str:
|
||||
prefix = f"/{self.api_prefix}" if self.api_prefix else ""
|
||||
return f"{self.base_url}{prefix}/{path.lstrip('/')}"
|
||||
|
||||
def _headers(self, auth: bool = True) -> dict[str, str]:
|
||||
headers: dict[str, str] = {
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "SmartUp/1.0",
|
||||
}
|
||||
if not auth:
|
||||
return headers
|
||||
if self.auth_type == "bearer":
|
||||
token = self.auth_config.get("token", "")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
elif self.auth_type == "api_key":
|
||||
key = self.auth_config.get("key", "")
|
||||
header = self.auth_config.get("header", "Authorization")
|
||||
if key:
|
||||
headers[header] = key
|
||||
elif self.auth_type == "login_password" and self._token:
|
||||
headers["Authorization"] = f"Bearer {self._token}"
|
||||
return headers
|
||||
|
||||
def _request(self, method: str, path: str, body: Any = None, auth: bool = True) -> Any:
|
||||
url = self._url(path)
|
||||
with httpx.Client(timeout=self.timeout) as client:
|
||||
if body is not None:
|
||||
resp = client.request(method, url, json=body, headers=self._headers(auth))
|
||||
else:
|
||||
resp = client.request(method, url, headers=self._headers(auth))
|
||||
resp.raise_for_status()
|
||||
ct = resp.headers.get("content-type", "")
|
||||
if not resp.content:
|
||||
return None
|
||||
text = resp.text
|
||||
if "application/json" not in ct and text.lstrip().startswith("<"):
|
||||
raise UpstreamError(f"{method} {path} returned HTML, not JSON")
|
||||
return resp.json()
|
||||
|
||||
def login(self) -> None:
|
||||
if self.auth_type != "login_password":
|
||||
return
|
||||
email = self.auth_config.get("email", "")
|
||||
password = self.auth_config.get("password", "")
|
||||
login_path = self.auth_config.get("login_path", "/auth/login")
|
||||
if not email or not password:
|
||||
raise UpstreamError("login_password auth requires email and password in auth_config")
|
||||
resp = self._request("POST", login_path, {"email": email, "password": password}, auth=False)
|
||||
token = _find_token(resp)
|
||||
if not token:
|
||||
raise UpstreamError("login succeeded but no token found in response")
|
||||
self._token = token
|
||||
|
||||
def get_available_groups(self, endpoint: str) -> list[dict[str, Any]]:
|
||||
resp = self._request("GET", endpoint)
|
||||
groups = _unwrap_list(resp)
|
||||
if groups is None:
|
||||
raise UpstreamError(f"{endpoint} did not return a list")
|
||||
return groups
|
||||
|
||||
def get_group_rates(self, endpoint: str) -> Any:
|
||||
return self._request("GET", endpoint)
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Send webhook notifications and write notification logs."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models.webhook_config import WebhookConfig
|
||||
from app.models.notification_log import NotificationLog
|
||||
from app.utils.dingtalk import (
|
||||
dingtalk_signed_url,
|
||||
format_dingtalk_rate_changed,
|
||||
format_dingtalk_status,
|
||||
)
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def _log(
|
||||
db: Session,
|
||||
webhook: WebhookConfig,
|
||||
event_type: str,
|
||||
payload: dict[str, Any],
|
||||
status: str,
|
||||
response_text: str,
|
||||
) -> None:
|
||||
entry = NotificationLog(
|
||||
webhook_config_id=webhook.id,
|
||||
webhook_name=webhook.name,
|
||||
event_type=event_type,
|
||||
payload_json=json.dumps(payload, ensure_ascii=False),
|
||||
status=status,
|
||||
response_text=response_text[:2000] if response_text else None,
|
||||
)
|
||||
db.add(entry)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _send_generic(url: str, payload: dict[str, Any], timeout: float = 15.0) -> str:
|
||||
resp = httpx.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json", "User-Agent": "SmartUp/1.0"},
|
||||
timeout=timeout,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.text[:500]
|
||||
|
||||
|
||||
def _send_dingtalk(url: str, secret: str, payload: dict[str, Any], timeout: float = 15.0) -> str:
|
||||
signed = dingtalk_signed_url(url, secret) if secret else url
|
||||
resp = httpx.post(
|
||||
signed,
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json", "User-Agent": "SmartUp/1.0"},
|
||||
timeout=timeout,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
result = resp.json()
|
||||
if result.get("errcode", 0) != 0:
|
||||
raise RuntimeError(f"DingTalk error: {resp.text[:300]}")
|
||||
return resp.text[:500]
|
||||
|
||||
|
||||
def send_rate_changed(
|
||||
db: Session,
|
||||
upstream_id: int,
|
||||
upstream_name: str,
|
||||
base_url: str,
|
||||
changes: list[dict[str, Any]],
|
||||
) -> None:
|
||||
webhooks = (
|
||||
db.query(WebhookConfig)
|
||||
.filter(WebhookConfig.enabled == True)
|
||||
.all()
|
||||
)
|
||||
changed_at = _now_iso()
|
||||
generic_payload = {
|
||||
"event": "upstream_rate_changed",
|
||||
"upstream": {"id": upstream_id, "name": upstream_name, "base_url": base_url},
|
||||
"changed_at": changed_at,
|
||||
"changes": changes,
|
||||
}
|
||||
for wh in webhooks:
|
||||
events = json.loads(wh.events_json or "[]")
|
||||
if "upstream_rate_changed" not in events:
|
||||
continue
|
||||
try:
|
||||
if wh.type == "dingtalk":
|
||||
msg = format_dingtalk_rate_changed(upstream_name, changed_at, changes)
|
||||
resp_text = _send_dingtalk(wh.url, wh.secret, msg)
|
||||
else:
|
||||
resp_text = _send_generic(wh.url, generic_payload)
|
||||
_log(db, wh, "upstream_rate_changed", generic_payload, "success", resp_text)
|
||||
except Exception as exc:
|
||||
_log(db, wh, "upstream_rate_changed", generic_payload, "failed", str(exc))
|
||||
|
||||
|
||||
def send_status_event(
|
||||
db: Session,
|
||||
upstream_id: int,
|
||||
upstream_name: str,
|
||||
base_url: str,
|
||||
event: str,
|
||||
error: str = "",
|
||||
) -> None:
|
||||
webhooks = (
|
||||
db.query(WebhookConfig)
|
||||
.filter(WebhookConfig.enabled == True)
|
||||
.all()
|
||||
)
|
||||
changed_at = _now_iso()
|
||||
generic_payload = {
|
||||
"event": event,
|
||||
"upstream": {"id": upstream_id, "name": upstream_name, "base_url": base_url},
|
||||
"changed_at": changed_at,
|
||||
"error": error,
|
||||
}
|
||||
for wh in webhooks:
|
||||
events = json.loads(wh.events_json or "[]")
|
||||
if event not in events:
|
||||
continue
|
||||
try:
|
||||
if wh.type == "dingtalk":
|
||||
msg = format_dingtalk_status(upstream_name, event, changed_at, error)
|
||||
resp_text = _send_dingtalk(wh.url, wh.secret, msg)
|
||||
else:
|
||||
resp_text = _send_generic(wh.url, generic_payload)
|
||||
_log(db, wh, event, generic_payload, "success", resp_text)
|
||||
except Exception as exc:
|
||||
_log(db, wh, event, generic_payload, "failed", str(exc))
|
||||
|
||||
|
||||
def send_test_notification(db: Session, webhook: WebhookConfig) -> tuple[bool, str]:
|
||||
payload = {
|
||||
"event": "test",
|
||||
"message": "SmartUp webhook test notification",
|
||||
"sent_at": _now_iso(),
|
||||
}
|
||||
try:
|
||||
if webhook.type == "dingtalk":
|
||||
msg = {
|
||||
"msgtype": "text",
|
||||
"text": {"content": "✅ SmartUp webhook 测试通知\n配置正常，连接成功。"},
|
||||
}
|
||||
resp_text = _send_dingtalk(webhook.url, webhook.secret, msg)
|
||||
else:
|
||||
resp_text = _send_generic(webhook.url, payload)
|
||||
_log(db, webhook, "test", payload, "success", resp_text)
|
||||
return True, "发送成功"
|
||||
except Exception as exc:
|
||||
_log(db, webhook, "test", payload, "failed", str(exc))
|
||||
return False, str(exc)
|
||||
Reference in New Issue
Block a user