fix: complete remaining 8 optimization items

- HTTP connection pooling: UpstreamClient & WebsiteClient reuse httpx.Client
- Deduplicate decimal_string into shared app/utils/number.py
- Split scheduler transaction: snapshot write → webhook/website sync in separate sessions
- Remove hardcoded 170.106.100.210 migration from database.py
- Reset consecutive_failures on upstream update
- Healthcheck: install curl, replace python -c with curl -f
- Add .dockerignore to reduce build context
- Frontend: add axios-retry with exponential backoff (5xx/network errors only)
This commit is contained in:
SmartUp Developer
2026-05-17 11:09:35 +08:00
parent ad16618406
commit 8a6ed249be
12 changed files with 211 additions and 89 deletions
+30
View File
@@ -0,0 +1,30 @@
# dependencies
node_modules/
backend/__pycache__/
backend/**/__pycache__/
backend/*.py[cod]
.venv/
venv/
# data & runtime
data/
*.log
# git
.git/
.gitignore
.gitattributes
# IDE
.idea/
.vscode/
*.swp
*.swo
# OS
.DS_Store
Thumbs.db
# env
.env
.env.*
+1
View File
@@ -25,6 +25,7 @@ RUN apt-get update \
libfreetype6 libgbm1 libglib2.0-0t64 libgtk-3-0t64 libnspr4 libnss3 \ libfreetype6 libgbm1 libglib2.0-0t64 libgtk-3-0t64 libnspr4 libnss3 \
libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 \ libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 \
libxdamage1 libxext6 libxfixes3 libxrandr2 libxshmfence1 xvfb \ libxdamage1 libxext6 libxfixes3 libxrandr2 libxshmfence1 xvfb \
curl \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
RUN playwright install chromium RUN playwright install chromium
+1 -12
View File
@@ -64,15 +64,4 @@ def _migrate_custom_pages():
"AND NULLIF(TRIM(login_password), '') IS NOT NULL" "AND NULLIF(TRIM(login_password), '') IS NOT NULL"
) )
) )
conn.execute(
text(
"UPDATE custom_pages "
"SET access_mode = 'remote_browser', use_proxy = 0 "
"WHERE url LIKE :host OR url LIKE :host_slash OR url LIKE :host_port"
),
{
"host": "%://170.106.100.210",
"host_slash": "%://170.106.100.210/%",
"host_port": "%://170.106.100.210:%",
},
)
+2
View File
@@ -121,6 +121,8 @@ def update_upstream(
data["base_url"] = data["base_url"].rstrip("/") data["base_url"] = data["base_url"].rstrip("/")
for k, v in data.items(): for k, v in data.items():
setattr(u, k, v) setattr(u, k, v)
# Reset failure counter on any update — the user may have fixed the issue
u.consecutive_failures = 0
u.updated_at = datetime.now(timezone.utc) u.updated_at = datetime.now(timezone.utc)
db.commit() db.commit()
db.refresh(u) db.refresh(u)
+74 -22
View File
@@ -28,9 +28,16 @@ def get_scheduler() -> BackgroundScheduler:
def _check_upstream(upstream_id: int) -> None: def _check_upstream(upstream_id: int) -> None:
"""Full upstream check executed by scheduler (runs in thread).""" """Full upstream check executed by scheduler (runs in thread).
Phase 1 — upstream API call + snapshot write (single transaction).
Phase 2 — webhook/website sync (separate sessions, so a notification
failure never rolls back the snapshot).
"""
settings = get_settings() settings = get_settings()
# ── Phase 1: upstream check + DB write ──────────────────────────
db: Session = SessionLocal() db: Session = SessionLocal()
client = None
try: try:
upstream = db.query(Upstream).filter(Upstream.id == upstream_id).first() upstream = db.query(Upstream).filter(Upstream.id == upstream_id).first()
if not upstream or not upstream.enabled: if not upstream or not upstream.enabled:
@@ -47,6 +54,8 @@ def _check_upstream(upstream_id: int) -> None:
) )
was_unhealthy = upstream.last_status == "unhealthy" was_unhealthy = upstream.last_status == "unhealthy"
snapshot = None
changes = None
try: try:
client.login() client.login()
@@ -61,16 +70,18 @@ def _check_upstream(upstream_id: int) -> None:
upstream.last_error = str(exc) upstream.last_error = str(exc)
upstream.last_checked_at = datetime.now(timezone.utc) upstream.last_checked_at = datetime.now(timezone.utc)
threshold = settings.unhealthy_threshold threshold = settings.unhealthy_threshold
if upstream.consecutive_failures >= threshold and upstream.last_status != "unhealthy": became_unhealthy = (
upstream.consecutive_failures >= threshold
and upstream.last_status != "unhealthy"
)
if became_unhealthy:
upstream.last_status = "unhealthy" upstream.last_status = "unhealthy"
db.commit() db.commit()
webhook_service.send_status_event(
db, upstream.id, upstream.name, upstream.base_url,
"upstream_unhealthy", str(exc)
)
else:
db.commit()
logger.warning("upstream %s check failed: %s", upstream.name, exc) logger.warning("upstream %s check failed: %s", upstream.name, exc)
# Phase 2: notify unhealthy in a fresh session
if became_unhealthy:
_notify_status(upstream.id, upstream.name, upstream.base_url,
"upstream_unhealthy", str(exc))
return return
# success path # success path
@@ -90,29 +101,70 @@ def _check_upstream(upstream_id: int) -> None:
captured_at=datetime.now(timezone.utc), captured_at=datetime.now(timezone.utc),
) )
db.add(new_row) db.add(new_row)
prune_snapshots(db, upstream_id, settings.snapshot_retention_count)
# update upstream status # update upstream status
upstream.last_status = "healthy" upstream.last_status = "healthy"
upstream.last_checked_at = datetime.now(timezone.utc) upstream.last_checked_at = datetime.now(timezone.utc)
upstream.last_error = None upstream.last_error = None
upstream.consecutive_failures = 0 upstream.consecutive_failures = 0
prune_snapshots(db, upstream_id, settings.snapshot_retention_count)
db.commit() db.commit()
if was_unhealthy: logger.info(
webhook_service.send_status_event( "upstream %s: %d rate change(s)" if changes else "upstream %s: no changes",
db, upstream.id, upstream.name, upstream.base_url, "upstream_recovered" upstream.name, len(changes) if changes else 0,
) )
if changes: finally:
webhook_service.send_rate_changed( client.close()
db, upstream.id, upstream.name, upstream.base_url, changes db.close()
)
website_sync.sync_affected_bindings(db, upstream.id, changes)
logger.info("upstream %s: %d rate change(s)", upstream.name, len(changes))
else:
logger.debug("upstream %s: no changes", upstream.name)
# ── Phase 2: notifications (independent sessions) ──────────────
if was_unhealthy:
_notify_status(upstream_id, upstream.name, upstream.base_url, "upstream_recovered")
if changes:
_notify_rate_changed(upstream_id, upstream.name, upstream.base_url, changes)
_sync_website_bindings(upstream_id, changes)
def _notify_status(
upstream_id: int,
upstream_name: str,
base_url: str,
event: str,
error: str = "",
) -> None:
db = SessionLocal()
try:
webhook_service.send_status_event(db, upstream_id, upstream_name, base_url, event, error)
except Exception:
logger.exception("status webhook failed for upstream %s", upstream_name)
finally:
db.close()
def _notify_rate_changed(
upstream_id: int,
upstream_name: str,
base_url: str,
changes: list[dict[str, Any]],
) -> None:
db = SessionLocal()
try:
webhook_service.send_rate_changed(db, upstream_id, upstream_name, base_url, changes)
except Exception:
logger.exception("rate webhook failed for upstream %s", upstream_name)
finally:
db.close()
def _sync_website_bindings(upstream_id: int, changes: list[dict[str, Any]]) -> None:
db = SessionLocal()
try:
website_sync.sync_affected_bindings(db, upstream_id, changes)
except Exception:
logger.exception("website sync failed for upstream %s", upstream_id)
finally: finally:
db.close() db.close()
+25 -34
View File
@@ -2,11 +2,12 @@
from __future__ import annotations from __future__ import annotations
import json import json
from decimal import Decimal, InvalidOperation
from typing import Any, Optional from typing import Any, Optional
from urllib.parse import urljoin from urllib.parse import urljoin
import httpx import httpx
from app.utils.number import decimal_string
class UpstreamError(RuntimeError): class UpstreamError(RuntimeError):
pass pass
@@ -66,19 +67,6 @@ def _unwrap_list(value: Any) -> Optional[list[dict[str, Any]]]:
return None return None
def _decimal_str(value: Any) -> str:
if value is None or value == "":
return ""
try:
d = Decimal(str(value))
except (InvalidOperation, ValueError):
return str(value)
n = d.normalize()
if n == n.to_integral():
return str(n.quantize(Decimal("1")))
return format(n, "f")
def _group_id(group: dict[str, Any]) -> str: def _group_id(group: dict[str, Any]) -> str:
for key in ("id", "group_id", "groupId"): for key in ("id", "group_id", "groupId"):
v = group.get(key) v = group.get(key)
@@ -117,7 +105,7 @@ def _extract_rates_map(raw: Any) -> dict[str, str]:
if isinstance(parsed, dict): if isinstance(parsed, dict):
result: dict[str, str] = {} result: dict[str, str] = {}
for k, v in parsed.items(): for k, v in parsed.items():
r = _decimal_str(v) r = decimal_string(v)
if r: if r:
result[str(k)] = r result[str(k)] = r
return result return result
@@ -127,7 +115,7 @@ def _extract_rates_map(raw: Any) -> dict[str, str]:
# In case it's returned as dict directly # In case it's returned as dict directly
result = {} result = {}
for k, v in val.items(): for k, v in val.items():
r = _decimal_str(v) r = decimal_string(v)
if r: if r:
result[str(k)] = r result[str(k)] = r
return result return result
@@ -153,13 +141,13 @@ def _extract_rates_map(raw: Any) -> dict[str, str]:
result: dict[str, str] = {} result: dict[str, str] = {}
for k, v in candidates.items(): for k, v in candidates.items():
if isinstance(v, dict): if isinstance(v, dict):
r = _decimal_str( r = decimal_string(
v.get("rate_multiplier") or v.get("rateMultiplier") v.get("rate_multiplier") or v.get("rateMultiplier")
or v.get("user_rate_multiplier") or v.get("userRateMultiplier") or v.get("user_rate_multiplier") or v.get("userRateMultiplier")
or v.get("ratio") or v.get("ratio")
) )
else: else:
r = _decimal_str(v) r = decimal_string(v)
if r: if r:
result[str(k)] = r result[str(k)] = r
return result return result
@@ -221,6 +209,10 @@ class UpstreamClient:
self._token: str = "" self._token: str = ""
self._cookies: dict[str, str] = {} self._cookies: dict[str, str] = {}
self._new_api_user: str = "" self._new_api_user: str = ""
self._client = httpx.Client(timeout=timeout)
def close(self) -> None:
self._client.close()
def _url(self, path: str) -> str: def _url(self, path: str) -> str:
prefix = f"/{self.api_prefix}" if self.api_prefix else "" prefix = f"/{self.api_prefix}" if self.api_prefix else ""
@@ -250,22 +242,21 @@ class UpstreamClient:
def _request(self, method: str, path: str, body: Any = None, auth: bool = True) -> Any: def _request(self, method: str, path: str, body: Any = None, auth: bool = True) -> Any:
url = self._url(path) url = self._url(path)
with httpx.Client(timeout=self.timeout) as client: if body is not None:
if body is not None: resp = self._client.request(
resp = client.request( method,
method, url,
url, json=body,
json=body, headers=self._headers(auth),
headers=self._headers(auth), cookies=self._cookies,
cookies=self._cookies, )
) else:
else: resp = self._client.request(
resp = client.request( method,
method, url,
url, headers=self._headers(auth),
headers=self._headers(auth), cookies=self._cookies,
cookies=self._cookies, )
)
self._cookies.update(dict(resp.cookies)) self._cookies.update(dict(resp.cookies))
resp.raise_for_status() resp.raise_for_status()
ct = resp.headers.get("content-type", "") ct = resp.headers.get("content-type", "")
+7 -15
View File
@@ -6,24 +6,13 @@ from urllib.parse import quote
import httpx import httpx
from app.utils.number import decimal_string
class WebsiteError(RuntimeError): class WebsiteError(RuntimeError):
pass pass
def decimal_string(value: Any) -> str:
if value is None or value == "":
return ""
try:
d = Decimal(str(value))
except (InvalidOperation, ValueError):
return str(value)
n = d.normalize()
if n == n.to_integral():
return str(n.quantize(Decimal("1")))
return format(n, "f")
def parse_positive_decimal(value: Any) -> Decimal | None: def parse_positive_decimal(value: Any) -> Decimal | None:
if value is None or value == "": if value is None or value == "":
return None return None
@@ -111,6 +100,10 @@ class Sub2ApiWebsiteClient:
self.auth_type = auth_type self.auth_type = auth_type
self.auth_config = auth_config self.auth_config = auth_config
self.timeout = timeout self.timeout = timeout
self._client = httpx.Client(timeout=timeout)
def close(self) -> None:
self._client.close()
def _url(self, path: str) -> str: def _url(self, path: str) -> str:
prefix = f"/{self.api_prefix}" if self.api_prefix else "" prefix = f"/{self.api_prefix}" if self.api_prefix else ""
@@ -130,8 +123,7 @@ class Sub2ApiWebsiteClient:
return headers return headers
def _request(self, method: str, path: str, body: Any = None) -> Any: def _request(self, method: str, path: str, body: Any = None) -> Any:
with httpx.Client(timeout=self.timeout) as client: resp = self._client.request(method, self._url(path), json=body, headers=self._headers())
resp = client.request(method, self._url(path), json=body, headers=self._headers())
resp.raise_for_status() resp.raise_for_status()
if not resp.content: if not resp.content:
return None return None
+25
View File
@@ -0,0 +1,25 @@
"""Shared numeric formatting utilities."""
from __future__ import annotations
from decimal import Decimal, InvalidOperation
from typing import Any
def decimal_string(value: Any) -> str:
"""Format a numeric value as a clean decimal string.
- None / empty → ""
- Whole numbers → no decimal point (e.g. "5")
- Decimals → trailing zeros stripped (e.g. "3.14")
- Unparseable → raw str()
"""
if value is None or value == "":
return ""
try:
d = Decimal(str(value))
except (InvalidOperation, ValueError):
return str(value)
n = d.normalize()
if n == n.to_integral():
return str(n.quantize(Decimal("1")))
return format(n, "f")
+1 -1
View File
@@ -17,7 +17,7 @@ services:
- TZ=${TZ:-Asia/Shanghai} - TZ=${TZ:-Asia/Shanghai}
- UNHEALTHY_THRESHOLD=${UNHEALTHY_THRESHOLD:-3} - UNHEALTHY_THRESHOLD=${UNHEALTHY_THRESHOLD:-3}
healthcheck: healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz', timeout=5).read()"] test: ["CMD", "curl", "-f", "http://localhost:8000/healthz"]
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 3 retries: 3
+25
View File
@@ -10,6 +10,7 @@
"dependencies": { "dependencies": {
"@element-plus/icons-vue": "^2.3.1", "@element-plus/icons-vue": "^2.3.1",
"axios": "^1.7.9", "axios": "^1.7.9",
"axios-retry": "^4.5.0",
"dayjs": "^1.11.13", "dayjs": "^1.11.13",
"element-plus": "^2.8.8", "element-plus": "^2.8.8",
"pinia": "^2.2.6", "pinia": "^2.2.6",
@@ -1202,6 +1203,18 @@
"proxy-from-env": "^2.1.0" "proxy-from-env": "^2.1.0"
} }
}, },
"node_modules/axios-retry": {
"version": "4.5.0",
"resolved": "https://registry.npmmirror.com/axios-retry/-/axios-retry-4.5.0.tgz",
"integrity": "sha512-aR99oXhpEDGo0UuAlYcn2iGRds30k366Zfa05XWScR9QaQD4JYiP3/1Qt1u7YlefUOK+cn0CcwoL1oefavQUlQ==",
"license": "Apache-2.0",
"dependencies": {
"is-retry-allowed": "^2.2.0"
},
"peerDependencies": {
"axios": "0.x || 1.x"
}
},
"node_modules/balanced-match": { "node_modules/balanced-match": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmmirror.com/balanced-match/-/balanced-match-1.0.2.tgz", "resolved": "https://registry.npmmirror.com/balanced-match/-/balanced-match-1.0.2.tgz",
@@ -1593,6 +1606,18 @@
"he": "bin/he" "he": "bin/he"
} }
}, },
"node_modules/is-retry-allowed": {
"version": "2.2.0",
"resolved": "https://registry.npmmirror.com/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz",
"integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==",
"license": "MIT",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/lodash": { "node_modules/lodash": {
"version": "4.18.1", "version": "4.18.1",
"resolved": "https://registry.npmmirror.com/lodash/-/lodash-4.18.1.tgz", "resolved": "https://registry.npmmirror.com/lodash/-/lodash-4.18.1.tgz",
+6 -5
View File
@@ -8,13 +8,14 @@
"preview": "vite preview" "preview": "vite preview"
}, },
"dependencies": { "dependencies": {
"vue": "^3.5.13",
"vue-router": "^4.4.5",
"pinia": "^2.2.6",
"element-plus": "^2.8.8",
"@element-plus/icons-vue": "^2.3.1", "@element-plus/icons-vue": "^2.3.1",
"axios": "^1.7.9", "axios": "^1.7.9",
"dayjs": "^1.11.13" "axios-retry": "^4.5.0",
"dayjs": "^1.11.13",
"element-plus": "^2.8.8",
"pinia": "^2.2.6",
"vue": "^3.5.13",
"vue-router": "^4.4.5"
}, },
"devDependencies": { "devDependencies": {
"@vitejs/plugin-vue": "^5.2.1", "@vitejs/plugin-vue": "^5.2.1",
+14
View File
@@ -1,4 +1,5 @@
import axios from 'axios' import axios from 'axios'
import axiosRetry from 'axios-retry'
import router from '@/router' import router from '@/router'
import { authStorageKeys } from '@/authStorage' import { authStorageKeys } from '@/authStorage'
@@ -7,6 +8,19 @@ export const api = axios.create({
timeout: 30000, timeout: 30000,
}) })
axiosRetry(api, {
retries: 3,
retryDelay: axiosRetry.exponentialDelay,
retryCondition: (err) => {
// Retry on network errors or 5xx, but never on 401/403/404/4xx
if (!err.response) return true
return err.response.status >= 500 && err.response.status < 600
},
onRetry: (_retryCount, _err, _requestConfig) => {
// no-op — could log in dev
},
})
api.interceptors.response.use( api.interceptors.response.use(
(r) => r, (r) => r,
(err) => { (err) => {