LiuMangMang/SmartUp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: complete remaining 8 optimization items

Browse Source 
- HTTP connection pooling: UpstreamClient & WebsiteClient reuse httpx.Client
- Deduplicate decimal_string into shared app/utils/number.py
- Split scheduler transaction: snapshot write → webhook/website sync in separate sessions
- Remove hardcoded 170.106.100.210 migration from database.py
- Reset consecutive_failures on upstream update
- Healthcheck: install curl, replace python -c with curl -f
- Add .dockerignore to reduce build context
- Frontend: add axios-retry with exponential backoff (5xx/network errors only)
This commit is contained in:
SmartUp Developer
2026-05-17 11:09:35 +08:00
parent ad16618406
commit 8a6ed249be
12 changed files with 211 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+30
View File
@@ -0,0 +1,30 @@
# dependencies
node_modules/
backend/__pycache__/
backend/**/__pycache__/
backend/*.py[cod]
.venv/
venv/
# data & runtime
data/
*.log
# git
.git/
.gitignore
.gitattributes
# IDE
.idea/
.vscode/
*.swp
*.swo
# OS
.DS_Store
Thumbs.db
# env
.env
.env.*
Dockerfile
+1
View File
@@ -25,6 +25,7 @@ RUN apt-get update \
libfreetype6 libgbm1 libglib2.0-0t64 libgtk-3-0t64 libnspr4 libnss3 \
libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 \
libxdamage1 libxext6 libxfixes3 libxrandr2 libxshmfence1 xvfb \
curl \
&& rm -rf /var/lib/apt/lists/*
RUN playwright install chromium
backend/app/database.py
+1 -12
View File
@@ -64,15 +64,4 @@ def _migrate_custom_pages():
"AND NULLIF(TRIM(login_password), '') IS NOT NULL"
)
)
conn.execute(
text(
"UPDATE custom_pages "
"SET access_mode = 'remote_browser', use_proxy = 0 "
"WHERE url LIKE :host OR url LIKE :host_slash OR url LIKE :host_port"
),
{
"host": "%://170.106.100.210",
"host_slash": "%://170.106.100.210/%",
"host_port": "%://170.106.100.210:%",
},
)
backend/app/routers/upstreams.py
+2
View File
@@ -121,6 +121,8 @@ def update_upstream(
data["base_url"] = data["base_url"].rstrip("/")
for k, v in data.items():
setattr(u, k, v)
# Reset failure counter on any update — the user may have fixed the issue
u.consecutive_failures = 0
u.updated_at = datetime.now(timezone.utc)
db.commit()
db.refresh(u)
backend/app/services/scheduler.py
+74 -22
View File
@@ -28,9 +28,16 @@ def get_scheduler() -> BackgroundScheduler:
def _check_upstream(upstream_id: int) -> None:
"""Full upstream check executed by scheduler (runs in thread)."""
"""Full upstream check executed by scheduler (runs in thread).
Phase 1 — upstream API call + snapshot write (single transaction).
Phase 2 — webhook/website sync (separate sessions, so a notification
failure never rolls back the snapshot).
"""
settings = get_settings()
# ── Phase 1: upstream check + DB write ──────────────────────────
db: Session = SessionLocal()
client = None
try:
upstream = db.query(Upstream).filter(Upstream.id == upstream_id).first()
if not upstream or not upstream.enabled:
@@ -47,6 +54,8 @@ def _check_upstream(upstream_id: int) -> None:
)
was_unhealthy = upstream.last_status == "unhealthy"
snapshot = None
changes = None
try:
client.login()
@@ -61,16 +70,18 @@ def _check_upstream(upstream_id: int) -> None:
upstream.last_error = str(exc)
upstream.last_checked_at = datetime.now(timezone.utc)
threshold = settings.unhealthy_threshold
if upstream.consecutive_failures >= threshold and upstream.last_status != "unhealthy":
became_unhealthy = (
upstream.consecutive_failures >= threshold
and upstream.last_status != "unhealthy"
)
if became_unhealthy:
upstream.last_status = "unhealthy"
db.commit()
webhook_service.send_status_event(
db, upstream.id, upstream.name, upstream.base_url,
"upstream_unhealthy", str(exc)
)
else:
db.commit()
db.commit()
logger.warning("upstream %s check failed: %s", upstream.name, exc)
# Phase 2: notify unhealthy in a fresh session
if became_unhealthy:
_notify_status(upstream.id, upstream.name, upstream.base_url,
"upstream_unhealthy", str(exc))
return
# success path
@@ -90,29 +101,70 @@ def _check_upstream(upstream_id: int) -> None:
captured_at=datetime.now(timezone.utc),
)
db.add(new_row)
prune_snapshots(db, upstream_id, settings.snapshot_retention_count)
# update upstream status
upstream.last_status = "healthy"
upstream.last_checked_at = datetime.now(timezone.utc)
upstream.last_error = None
upstream.consecutive_failures = 0
prune_snapshots(db, upstream_id, settings.snapshot_retention_count)
db.commit()
if was_unhealthy:
webhook_service.send_status_event(
db, upstream.id, upstream.name, upstream.base_url, "upstream_recovered"
)
logger.info(
"upstream %s: %d rate change(s)" if changes else "upstream %s: no changes",
upstream.name, len(changes) if changes else 0,
)
if changes:
webhook_service.send_rate_changed(
db, upstream.id, upstream.name, upstream.base_url, changes
)
website_sync.sync_affected_bindings(db, upstream.id, changes)
logger.info("upstream %s: %d rate change(s)", upstream.name, len(changes))
else:
logger.debug("upstream %s: no changes", upstream.name)
finally:
client.close()
db.close()
# ── Phase 2: notifications (independent sessions) ──────────────
if was_unhealthy:
_notify_status(upstream_id, upstream.name, upstream.base_url, "upstream_recovered")
if changes:
_notify_rate_changed(upstream_id, upstream.name, upstream.base_url, changes)
_sync_website_bindings(upstream_id, changes)
def _notify_status(
upstream_id: int,
upstream_name: str,
base_url: str,
event: str,
error: str = "",
) -> None:
db = SessionLocal()
try:
webhook_service.send_status_event(db, upstream_id, upstream_name, base_url, event, error)
except Exception:
logger.exception("status webhook failed for upstream %s", upstream_name)
finally:
db.close()
def _notify_rate_changed(
upstream_id: int,
upstream_name: str,
base_url: str,
changes: list[dict[str, Any]],
) -> None:
db = SessionLocal()
try:
webhook_service.send_rate_changed(db, upstream_id, upstream_name, base_url, changes)
except Exception:
logger.exception("rate webhook failed for upstream %s", upstream_name)
finally:
db.close()
def _sync_website_bindings(upstream_id: int, changes: list[dict[str, Any]]) -> None:
db = SessionLocal()
try:
website_sync.sync_affected_bindings(db, upstream_id, changes)
except Exception:
logger.exception("website sync failed for upstream %s", upstream_id)
finally:
db.close()
backend/app/services/upstream_client.py
+25 -34
View File
@@ -2,11 +2,12 @@
from __future__ import annotations
import json
from decimal import Decimal, InvalidOperation
from typing import Any, Optional
from urllib.parse import urljoin
import httpx
from app.utils.number import decimal_string
class UpstreamError(RuntimeError):
pass
@@ -66,19 +67,6 @@ def _unwrap_list(value: Any) -> Optional[list[dict[str, Any]]]:
return None
def _decimal_str(value: Any) -> str:
if value is None or value == "":
return ""
try:
d = Decimal(str(value))
except (InvalidOperation, ValueError):
return str(value)
n = d.normalize()
if n == n.to_integral():
return str(n.quantize(Decimal("1")))
return format(n, "f")
def _group_id(group: dict[str, Any]) -> str:
for key in ("id", "group_id", "groupId"):
v = group.get(key)
@@ -117,7 +105,7 @@ def _extract_rates_map(raw: Any) -> dict[str, str]:
if isinstance(parsed, dict):
result: dict[str, str] = {}
for k, v in parsed.items():
r = _decimal_str(v)
r = decimal_string(v)
if r:
result[str(k)] = r
return result
@@ -127,7 +115,7 @@ def _extract_rates_map(raw: Any) -> dict[str, str]:
# In case it's returned as dict directly
result = {}
for k, v in val.items():
r = _decimal_str(v)
r = decimal_string(v)
if r:
result[str(k)] = r
return result
@@ -153,13 +141,13 @@ def _extract_rates_map(raw: Any) -> dict[str, str]:
result: dict[str, str] = {}
for k, v in candidates.items():
if isinstance(v, dict):
r = _decimal_str(
r = decimal_string(
v.get("rate_multiplier") or v.get("rateMultiplier")
or v.get("user_rate_multiplier") or v.get("userRateMultiplier")
or v.get("ratio")
)
else:
r = _decimal_str(v)
r = decimal_string(v)
if r:
result[str(k)] = r
return result
@@ -221,6 +209,10 @@ class UpstreamClient:
self._token: str = ""
self._cookies: dict[str, str] = {}
self._new_api_user: str = ""
self._client = httpx.Client(timeout=timeout)
def close(self) -> None:
self._client.close()
def _url(self, path: str) -> str:
prefix = f"/{self.api_prefix}" if self.api_prefix else ""
@@ -250,22 +242,21 @@ class UpstreamClient:
def _request(self, method: str, path: str, body: Any = None, auth: bool = True) -> Any:
url = self._url(path)
with httpx.Client(timeout=self.timeout) as client:
if body is not None:
resp = client.request(
method,
url,
json=body,
headers=self._headers(auth),
cookies=self._cookies,
)
else:
resp = client.request(
method,
url,
headers=self._headers(auth),
cookies=self._cookies,
)
if body is not None:
resp = self._client.request(
method,
url,
json=body,
headers=self._headers(auth),
cookies=self._cookies,
)
else:
resp = self._client.request(
method,
url,
headers=self._headers(auth),
cookies=self._cookies,
)
self._cookies.update(dict(resp.cookies))
resp.raise_for_status()
ct = resp.headers.get("content-type", "")
backend/app/services/website_client.py
+7 -15
View File
@@ -6,24 +6,13 @@ from urllib.parse import quote
import httpx
from app.utils.number import decimal_string
class WebsiteError(RuntimeError):
pass
def decimal_string(value: Any) -> str:
if value is None or value == "":
return ""
try:
d = Decimal(str(value))
except (InvalidOperation, ValueError):
return str(value)
n = d.normalize()
if n == n.to_integral():
return str(n.quantize(Decimal("1")))
return format(n, "f")
def parse_positive_decimal(value: Any) -> Decimal | None:
if value is None or value == "":
return None
@@ -111,6 +100,10 @@ class Sub2ApiWebsiteClient:
self.auth_type = auth_type
self.auth_config = auth_config
self.timeout = timeout
self._client = httpx.Client(timeout=timeout)
def close(self) -> None:
self._client.close()
def _url(self, path: str) -> str:
prefix = f"/{self.api_prefix}" if self.api_prefix else ""
@@ -130,8 +123,7 @@ class Sub2ApiWebsiteClient:
return headers
def _request(self, method: str, path: str, body: Any = None) -> Any:
with httpx.Client(timeout=self.timeout) as client:
resp = client.request(method, self._url(path), json=body, headers=self._headers())
resp = self._client.request(method, self._url(path), json=body, headers=self._headers())
resp.raise_for_status()
if not resp.content:
return None
backend/app/utils/number.py
+25
View File
@@ -0,0 +1,25 @@
"""Shared numeric formatting utilities."""
from __future__ import annotations
from decimal import Decimal, InvalidOperation
from typing import Any
def decimal_string(value: Any) -> str:
"""Format a numeric value as a clean decimal string.
- None / empty → ""
- Whole numbers → no decimal point (e.g. "5")
- Decimals → trailing zeros stripped (e.g. "3.14")
- Unparseable → raw str()
"""
if value is None or value == "":
return ""
try:
d = Decimal(str(value))
except (InvalidOperation, ValueError):
return str(value)
n = d.normalize()
if n == n.to_integral():
return str(n.quantize(Decimal("1")))
return format(n, "f")
docker-compose.yml
+1 -1
View File
@@ -17,7 +17,7 @@ services:
- TZ=${TZ:-Asia/Shanghai}
- UNHEALTHY_THRESHOLD=${UNHEALTHY_THRESHOLD:-3}
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz', timeout=5).read()"]
test: ["CMD", "curl", "-f", "http://localhost:8000/healthz"]
interval: 30s
timeout: 10s
retries: 3
frontend/package-lock.json
Generated
+25
View File
@@ -10,6 +10,7 @@
"dependencies": {
"@element-plus/icons-vue": "^2.3.1",
"axios": "^1.7.9",
"axios-retry": "^4.5.0",
"dayjs": "^1.11.13",
"element-plus": "^2.8.8",
"pinia": "^2.2.6",
@@ -1202,6 +1203,18 @@
"proxy-from-env": "^2.1.0"
}
},
"node_modules/axios-retry": {
"version": "4.5.0",
"resolved": "https://registry.npmmirror.com/axios-retry/-/axios-retry-4.5.0.tgz",
"integrity": "sha512-aR99oXhpEDGo0UuAlYcn2iGRds30k366Zfa05XWScR9QaQD4JYiP3/1Qt1u7YlefUOK+cn0CcwoL1oefavQUlQ==",
"license": "Apache-2.0",
"dependencies": {
"is-retry-allowed": "^2.2.0"
},
"peerDependencies": {
"axios": "0.x || 1.x"
}
},
"node_modules/balanced-match": {
"version": "1.0.2",
"resolved": "https://registry.npmmirror.com/balanced-match/-/balanced-match-1.0.2.tgz",
@@ -1593,6 +1606,18 @@
"he": "bin/he"
}
},
"node_modules/is-retry-allowed": {
"version": "2.2.0",
"resolved": "https://registry.npmmirror.com/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz",
"integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==",
"license": "MIT",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/lodash": {
"version": "4.18.1",
"resolved": "https://registry.npmmirror.com/lodash/-/lodash-4.18.1.tgz",
frontend/package.json
+6 -5
View File
@@ -8,13 +8,14 @@
"preview": "vite preview"
},
"dependencies": {
"vue": "^3.5.13",
"vue-router": "^4.4.5",
"pinia": "^2.2.6",
"element-plus": "^2.8.8",
"@element-plus/icons-vue": "^2.3.1",
"axios": "^1.7.9",
"dayjs": "^1.11.13"
"axios-retry": "^4.5.0",
"dayjs": "^1.11.13",
"element-plus": "^2.8.8",
"pinia": "^2.2.6",
"vue": "^3.5.13",
"vue-router": "^4.4.5"
},
"devDependencies": {
"@vitejs/plugin-vue": "^5.2.1",
frontend/src/api/index.ts
+14
View File
@@ -1,4 +1,5 @@
import axios from 'axios'
import axiosRetry from 'axios-retry'
import router from '@/router'
import { authStorageKeys } from '@/authStorage'
@@ -7,6 +8,19 @@ export const api = axios.create({
timeout: 30000,
})
axiosRetry(api, {
retries: 3,
retryDelay: axiosRetry.exponentialDelay,
retryCondition: (err) => {
// Retry on network errors or 5xx, but never on 401/403/404/4xx
if (!err.response) return true
return err.response.status >= 500 && err.response.status < 600
},
onRetry: (_retryCount, _err, _requestConfig) => {
// no-op — could log in dev
},
})
api.interceptors.response.use(
(r) => r,
(err) => {