Remove server remote browser support
This commit is contained in:
liumangmang
@@ -15,8 +15,6 @@ class Settings(BaseSettings):
|
||||
tz: str = "Asia/Shanghai"
|
||||
# consecutive failures before upstream goes unhealthy
|
||||
unhealthy_threshold: int = 3
|
||||
browser_profiles_dir: str = "/app/data/browser-profiles"
|
||||
browser_headless: bool = True
|
||||
|
||||
@property
|
||||
def cors_origin_list(self) -> list[str]:
|
||||
|
||||
+1
-4
@@ -15,8 +15,7 @@ from app.models.admin_user import AdminUser
|
||||
from app.database import SessionLocal
|
||||
from app.utils.auth import hash_password, verify_password, validate_password_supported
|
||||
from app.services.scheduler import start_scheduler, stop_scheduler
|
||||
from app.routers import auth, upstreams, webhooks, logs, custom_pages, browser_sessions, websites, auth_capture
|
||||
from app.services.browser_session_service import browser_sessions as browser_session_service
|
||||
from app.routers import auth, upstreams, webhooks, logs, custom_pages, websites, auth_capture
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -67,7 +66,6 @@ async def lifespan(app: FastAPI):
|
||||
_init_admin()
|
||||
start_scheduler()
|
||||
yield
|
||||
await browser_session_service.shutdown()
|
||||
stop_scheduler()
|
||||
|
||||
|
||||
@@ -97,7 +95,6 @@ app.include_router(upstreams.router)
|
||||
app.include_router(webhooks.router)
|
||||
app.include_router(logs.router)
|
||||
app.include_router(custom_pages.router)
|
||||
app.include_router(browser_sessions.router)
|
||||
app.include_router(websites.router)
|
||||
app.include_router(auth_capture.router)
|
||||
|
||||
|
||||
@@ -1,45 +1,23 @@
|
||||
"""Auth capture API — remote browser for manual login + credential extraction."""
|
||||
"""Auth capture API for real-browser credential imports."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.services.auth_capture_service import extract_all
|
||||
from app.services.browser_import_service import (
|
||||
ImportSessionError,
|
||||
browser_imports,
|
||||
build_import_result,
|
||||
)
|
||||
from app.services.browser_session_service import (
|
||||
BrowserDependencyError,
|
||||
BrowserSessionError,
|
||||
browser_sessions,
|
||||
)
|
||||
from app.utils.auth import get_current_user, get_user_from_token_param
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from app.utils.auth import get_current_user
|
||||
|
||||
router = APIRouter(prefix="/api/auth-capture", tags=["auth-capture"])
|
||||
|
||||
SENSITIVE_CANDIDATE_FIELDS = frozenset({"value", "cookie_value"})
|
||||
|
||||
|
||||
class CaptureSessionCreate(BaseModel):
|
||||
url: str = Field(..., description="Target login page URL to open in browser")
|
||||
width: int = Field(default=1280, ge=320, le=2560)
|
||||
height: int = Field(default=720, ge=240, le=1600)
|
||||
|
||||
|
||||
class CaptureSessionResponse(BaseModel):
|
||||
session_id: str
|
||||
ws_url: str
|
||||
|
||||
|
||||
class CaptureExtractResponse(BaseModel):
|
||||
cookies: list[dict] = []
|
||||
storage: dict[str, str] = {}
|
||||
@@ -82,91 +60,6 @@ def _sanitize_candidate(candidate: dict[str, Any]) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _browser_error(exc: Exception) -> HTTPException:
|
||||
if isinstance(exc, BrowserDependencyError):
|
||||
return HTTPException(503, str(exc))
|
||||
if isinstance(exc, BrowserSessionError):
|
||||
return HTTPException(409, str(exc))
|
||||
if isinstance(exc, KeyError):
|
||||
return HTTPException(404, "session not found")
|
||||
if isinstance(exc, ValueError):
|
||||
return HTTPException(400, str(exc))
|
||||
logger.exception("auth-capture error")
|
||||
return HTTPException(500, "internal error")
|
||||
|
||||
|
||||
def _ws_url(session_id: str, token: str) -> str:
|
||||
"""Build WebSocket URL for the remote browser viewer."""
|
||||
return f"/api/browser-sessions/{session_id}/ws?token={token}"
|
||||
|
||||
|
||||
@router.post("/sessions", response_model=CaptureSessionResponse, status_code=201)
|
||||
async def create_capture_session(
|
||||
body: CaptureSessionCreate,
|
||||
_=Depends(get_current_user),
|
||||
):
|
||||
"""Create a temporary browser session pointing at the given URL.
|
||||
|
||||
Returns a session_id and ws_url for the frontend to view/interact.
|
||||
The user should manually log in, then call GET /extract.
|
||||
"""
|
||||
try:
|
||||
session = await browser_sessions.create_ephemeral(
|
||||
url=body.url,
|
||||
width=body.width,
|
||||
height=body.height,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise _browser_error(exc)
|
||||
|
||||
# Build a short-lived token for WS auth (reuse current user's token logic)
|
||||
# The frontend already has the user's Bearer token, pass it via query param
|
||||
return CaptureSessionResponse(
|
||||
session_id=session.id,
|
||||
ws_url=f"/api/browser-sessions/{session.id}/ws",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/sessions/{session_id}/extract", response_model=CaptureExtractResponse)
|
||||
async def extract_credentials(
|
||||
session_id: str,
|
||||
include_raw: bool = Query(default=False, description="Include full cookies/storage/headers in response"),
|
||||
_=Depends(get_current_user),
|
||||
):
|
||||
"""Extract auth credentials from the browser session.
|
||||
|
||||
By default only returns curated candidates (typed, scored, with masked preview).
|
||||
Pass include_raw=true to also get full cookies, localStorage, and headers.
|
||||
"""
|
||||
try:
|
||||
session = browser_sessions.get_session(session_id)
|
||||
except KeyError:
|
||||
raise HTTPException(404, "session not found")
|
||||
|
||||
try:
|
||||
result = await extract_all(session)
|
||||
except Exception as exc:
|
||||
raise _browser_error(exc)
|
||||
|
||||
if not include_raw:
|
||||
# Strip raw data — only keep curated candidates with masked previews
|
||||
candidates = [_sanitize_candidate(candidate) for candidate in result.get("candidates", [])]
|
||||
return CaptureExtractResponse(candidates=candidates)
|
||||
return CaptureExtractResponse(**result)
|
||||
|
||||
|
||||
@router.delete("/sessions/{session_id}", status_code=204)
|
||||
async def close_capture_session(
|
||||
session_id: str,
|
||||
_=Depends(get_current_user),
|
||||
):
|
||||
"""Close and release the auth-capture browser session."""
|
||||
try:
|
||||
await browser_sessions.close(session_id)
|
||||
except Exception as exc:
|
||||
raise _browser_error(exc)
|
||||
|
||||
|
||||
@router.post("/import-sessions", response_model=ImportSessionCreateResponse, status_code=201)
|
||||
async def create_import_session(
|
||||
body: ImportSessionCreate,
|
||||
|
||||
@@ -1,417 +0,0 @@
|
||||
"""Remote browser session API."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Literal, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, WebSocket, WebSocketDisconnect
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models.custom_page import CustomPage
|
||||
from app.services.browser_session_service import (
|
||||
BrowserDependencyError,
|
||||
BrowserSessionError,
|
||||
browser_sessions,
|
||||
)
|
||||
from app.utils.auth import decode_token, get_current_user, get_user_from_token_param
|
||||
from app.database import SessionLocal
|
||||
from app.models.admin_user import AdminUser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/browser-sessions", tags=["browser-sessions"])
|
||||
|
||||
|
||||
class BrowserSessionCreate(BaseModel):
|
||||
custom_page_id: int
|
||||
width: int = Field(default=1280)
|
||||
height: int = Field(default=720)
|
||||
|
||||
|
||||
class BrowserTabResponse(BaseModel):
|
||||
id: str
|
||||
title: str
|
||||
url: str
|
||||
created_at: float
|
||||
|
||||
|
||||
class BrowserSessionResponse(BaseModel):
|
||||
id: str
|
||||
custom_page_id: int
|
||||
url: str
|
||||
title: str
|
||||
active_tab_id: Optional[str] = None
|
||||
tabs: Optional[list[BrowserTabResponse]] = None
|
||||
tab_revision: Optional[int] = 0
|
||||
|
||||
|
||||
class BrowserSelectionResponse(BaseModel):
|
||||
text: str
|
||||
|
||||
|
||||
class BrowserEvent(BaseModel):
|
||||
type: Literal["click", "dblclick", "mousemove", "mousedown", "mouseup", "type", "key", "scroll", "reload", "back", "forward", "resize"]
|
||||
x: Optional[float] = None
|
||||
y: Optional[float] = None
|
||||
button: Optional[Literal["left", "right", "middle"]] = "left"
|
||||
text: Optional[str] = None
|
||||
key: Optional[str] = None
|
||||
delta_x: Optional[float] = 0
|
||||
delta_y: Optional[float] = 0
|
||||
width: Optional[int] = None
|
||||
height: Optional[int] = None
|
||||
|
||||
|
||||
def _error_from_browser(exc: Exception) -> HTTPException:
|
||||
if isinstance(exc, BrowserDependencyError):
|
||||
return HTTPException(503, str(exc))
|
||||
if isinstance(exc, BrowserSessionError):
|
||||
return HTTPException(409, str(exc))
|
||||
if isinstance(exc, KeyError):
|
||||
return HTTPException(404, "browser session not found")
|
||||
if isinstance(exc, ValueError):
|
||||
return HTTPException(400, str(exc))
|
||||
return HTTPException(502, f"Browser error: {exc}")
|
||||
|
||||
|
||||
@router.post("", response_model=BrowserSessionResponse, status_code=201)
|
||||
async def create_session(
|
||||
body: BrowserSessionCreate,
|
||||
db: Session = Depends(get_db),
|
||||
_=Depends(get_current_user),
|
||||
):
|
||||
page = db.query(CustomPage).filter(CustomPage.id == body.custom_page_id).first()
|
||||
if not page or not page.enabled:
|
||||
raise HTTPException(404, "page not found")
|
||||
if page.access_mode != "remote_browser":
|
||||
raise HTTPException(400, "custom page is not configured for remote browser mode")
|
||||
login_config = {
|
||||
"enabled": page.login_autofill_enabled,
|
||||
"username": page.login_username,
|
||||
"password": page.login_password,
|
||||
"username_selector": page.login_username_selector,
|
||||
"password_selector": page.login_password_selector,
|
||||
"submit_selector": page.login_submit_selector,
|
||||
}
|
||||
try:
|
||||
session = await browser_sessions.create(page.id, page.url, body.width, body.height, login_config)
|
||||
return await browser_sessions.state(session.id)
|
||||
except Exception as exc:
|
||||
raise _error_from_browser(exc)
|
||||
|
||||
|
||||
@router.get("/{session_id}", response_model=BrowserSessionResponse)
|
||||
async def get_session(session_id: str, _=Depends(get_current_user)):
|
||||
try:
|
||||
return await browser_sessions.state(session_id)
|
||||
except Exception as exc:
|
||||
raise _error_from_browser(exc)
|
||||
|
||||
|
||||
@router.get("/{session_id}/screenshot")
|
||||
async def session_screenshot(session_id: str, _=Depends(get_user_from_token_param)):
|
||||
try:
|
||||
image = await browser_sessions.screenshot(session_id)
|
||||
except Exception as exc:
|
||||
raise _error_from_browser(exc)
|
||||
return Response(content=image, media_type="image/jpeg", headers={"Cache-Control": "no-store"})
|
||||
|
||||
|
||||
@router.post("/{session_id}/events", response_model=BrowserSessionResponse)
|
||||
async def send_event(session_id: str, body: BrowserEvent, _=Depends(get_current_user)):
|
||||
try:
|
||||
payload: dict[str, Any] = body.model_dump(exclude_none=True)
|
||||
event_type = payload.pop("type")
|
||||
return await browser_sessions.event(session_id, event_type, payload)
|
||||
except Exception as exc:
|
||||
raise _error_from_browser(exc)
|
||||
|
||||
|
||||
@router.post("/{session_id}/tabs/{tab_id}/activate", response_model=BrowserSessionResponse)
|
||||
async def activate_tab(session_id: str, tab_id: str, _=Depends(get_current_user)):
|
||||
try:
|
||||
return await browser_sessions.activate_tab(session_id, tab_id)
|
||||
except Exception as exc:
|
||||
raise _error_from_browser(exc)
|
||||
|
||||
|
||||
@router.delete("/{session_id}/tabs/{tab_id}", response_model=BrowserSessionResponse)
|
||||
async def close_tab(session_id: str, tab_id: str, _=Depends(get_current_user)):
|
||||
try:
|
||||
return await browser_sessions.close_tab(session_id, tab_id)
|
||||
except Exception as exc:
|
||||
raise _error_from_browser(exc)
|
||||
|
||||
|
||||
@router.get("/{session_id}/selection", response_model=BrowserSelectionResponse)
|
||||
async def get_selection(session_id: str, _=Depends(get_current_user)):
|
||||
try:
|
||||
return BrowserSelectionResponse(text=await browser_sessions.selected_text(session_id))
|
||||
except Exception as exc:
|
||||
raise _error_from_browser(exc)
|
||||
|
||||
|
||||
class BrowserClipboardResponse(BaseModel):
|
||||
text: Optional[str] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
@router.get("/{session_id}/clipboard", response_model=BrowserClipboardResponse)
|
||||
async def session_clipboard(session_id: str, _=Depends(get_current_user)):
|
||||
"""Read text from the remote browser's clipboard."""
|
||||
from fastapi.responses import JSONResponse
|
||||
try:
|
||||
text, error = await browser_sessions.read_clipboard(session_id)
|
||||
body: dict[str, Any] = {}
|
||||
if text:
|
||||
body["text"] = text
|
||||
elif error == "denied":
|
||||
body["error"] = "远程浏览器未授予剪贴板读取权限"
|
||||
elif error == "read_failed":
|
||||
body["error"] = "读取远程剪贴板时发生内部错误"
|
||||
else:
|
||||
if error:
|
||||
logger.warning("clipboard read error for %s: %s", session_id[:12], error)
|
||||
body["error"] = "远程剪贴板为空"
|
||||
return JSONResponse(content=body, headers={"Cache-Control": "no-store"})
|
||||
except Exception as exc:
|
||||
raise _error_from_browser(exc)
|
||||
|
||||
|
||||
class AutofillLoginResponse(BaseModel):
|
||||
success: bool
|
||||
message: str
|
||||
|
||||
|
||||
@router.post("/{session_id}/autofill-login", response_model=AutofillLoginResponse)
|
||||
async def autofill_login(session_id: str, _=Depends(get_current_user)):
|
||||
"""Manually trigger login autofill for the remote browser session.
|
||||
|
||||
Uses the linked custom page's saved credentials. Never returns passwords.
|
||||
"""
|
||||
try:
|
||||
session_state = await browser_sessions.state(session_id)
|
||||
except Exception as exc:
|
||||
raise _error_from_browser(exc)
|
||||
|
||||
from app.database import SessionLocal as _Db
|
||||
from app.models.custom_page import CustomPage
|
||||
db = _Db()
|
||||
try:
|
||||
page = db.query(CustomPage).filter(
|
||||
CustomPage.id == session_state["custom_page_id"]
|
||||
).first()
|
||||
if not page or not page.enabled:
|
||||
raise HTTPException(400, "linked custom page is not available")
|
||||
if page.access_mode != "remote_browser":
|
||||
raise HTTPException(400, "linked custom page is not in remote browser mode")
|
||||
if not page.login_autofill_enabled:
|
||||
return AutofillLoginResponse(success=False, message="该页面未启用自动填充登录")
|
||||
if not page.login_username or not page.login_password:
|
||||
return AutofillLoginResponse(success=False, message="该页面未保存账号密码")
|
||||
|
||||
login_config = {
|
||||
"enabled": True,
|
||||
"username": page.login_username,
|
||||
"password": page.login_password,
|
||||
"username_selector": page.login_username_selector,
|
||||
"password_selector": page.login_password_selector,
|
||||
"submit_selector": page.login_submit_selector,
|
||||
}
|
||||
|
||||
filled = await browser_sessions.autofill_login(session_id, login_config)
|
||||
if filled:
|
||||
return AutofillLoginResponse(success=True, message="已填入账号密码")
|
||||
return AutofillLoginResponse(
|
||||
success=False,
|
||||
message="未找到登录输入框，请先关闭弹窗或进入登录页后重试",
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@router.delete("/{session_id}", status_code=204)
|
||||
async def close_session(session_id: str, _=Depends(get_current_user)):
|
||||
await browser_sessions.close(session_id)
|
||||
|
||||
|
||||
@router.delete("/profiles/{custom_page_id}", status_code=204)
|
||||
async def clear_profile(custom_page_id: int, _=Depends(get_current_user)):
|
||||
"""Close active session for the page and delete its profile directory.
|
||||
|
||||
On next open the browser starts fresh, losing login state.
|
||||
"""
|
||||
from app.models.custom_page import CustomPage
|
||||
from app.database import SessionLocal as _Db
|
||||
db = _Db()
|
||||
try:
|
||||
page = db.query(CustomPage).filter(CustomPage.id == custom_page_id).first()
|
||||
if not page or not page.enabled:
|
||||
raise HTTPException(404, "custom page not found")
|
||||
if page.access_mode != "remote_browser":
|
||||
raise HTTPException(400, "custom page is not in remote browser mode")
|
||||
try:
|
||||
await browser_sessions.clear_profile(custom_page_id, page.url)
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(500, str(exc))
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
# ——— WebSocket stream ———
|
||||
# Frame interval & diff detection (tuned for CPU efficiency)
|
||||
_WS_MIN_INTERVAL = 0.15
|
||||
_WS_IDLE_INTERVAL = 1.00
|
||||
_WS_ACTIVE_INTERVAL = 0.20
|
||||
_WS_BACKOFF_INTERVAL = 2.00
|
||||
_WS_DEEP_IDLE_INTERVAL = 5.00
|
||||
_WS_ACTIVE_WINDOW = 1.25
|
||||
|
||||
|
||||
async def _ws_authenticate(token: Optional[str]) -> bool:
|
||||
"""Validate JWT token for WebSocket connections."""
|
||||
if not token:
|
||||
return False
|
||||
email = decode_token(token)
|
||||
if not email:
|
||||
return False
|
||||
db = SessionLocal()
|
||||
try:
|
||||
user = db.query(AdminUser).filter(AdminUser.email == email).first()
|
||||
return user is not None
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@router.websocket("/{session_id}/ws")
|
||||
async def session_ws(
|
||||
websocket: WebSocket,
|
||||
session_id: str,
|
||||
token: Optional[str] = Query(default=None),
|
||||
):
|
||||
"""WebSocket endpoint: pushes JPEG frames as binary, receives JSON event messages."""
|
||||
# Authenticate before accepting
|
||||
if not await _ws_authenticate(token):
|
||||
await websocket.close(code=4401)
|
||||
return
|
||||
|
||||
await websocket.accept()
|
||||
|
||||
# Track when a user event arrived so we can temporarily speed up
|
||||
last_event_at: float = 0.0
|
||||
last_frame_hash: str = ""
|
||||
unchanged_count = 0
|
||||
|
||||
# Task: receive events from client
|
||||
async def receive_loop():
|
||||
nonlocal last_event_at, unchanged_count
|
||||
try:
|
||||
while True:
|
||||
raw = await websocket.receive_text()
|
||||
try:
|
||||
msg = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
msg_type = msg.get("type")
|
||||
if not msg_type:
|
||||
continue
|
||||
payload: dict[str, Any] = {k: v for k, v in msg.items() if k != "type"}
|
||||
try:
|
||||
await browser_sessions.event(session_id, msg_type, payload, include_state=False)
|
||||
last_event_at = asyncio.get_event_loop().time()
|
||||
unchanged_count = 0
|
||||
except Exception as exc:
|
||||
logger.warning("ws event error: %s", exc)
|
||||
try:
|
||||
await websocket.send_json({"error": str(exc)})
|
||||
except Exception:
|
||||
pass
|
||||
except (WebSocketDisconnect, asyncio.CancelledError):
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.debug("ws receive_loop ended: %s", exc)
|
||||
|
||||
# Task: push screenshots
|
||||
async def push_loop():
|
||||
nonlocal last_frame_hash, unchanged_count
|
||||
last_tab_revision = -1
|
||||
try:
|
||||
while True:
|
||||
now = asyncio.get_event_loop().time()
|
||||
if (now - last_event_at) < _WS_ACTIVE_WINDOW:
|
||||
interval = _WS_ACTIVE_INTERVAL
|
||||
elif unchanged_count >= 9:
|
||||
interval = _WS_DEEP_IDLE_INTERVAL
|
||||
elif unchanged_count >= 3:
|
||||
interval = _WS_BACKOFF_INTERVAL
|
||||
else:
|
||||
interval = _WS_IDLE_INTERVAL
|
||||
|
||||
try:
|
||||
# Check for tab state changes
|
||||
session_obj = browser_sessions.get_session(session_id)
|
||||
if session_obj.tab_revision != last_tab_revision:
|
||||
last_tab_revision = session_obj.tab_revision
|
||||
state = await browser_sessions.state(session_id)
|
||||
await websocket.send_json({"type": "state", "session": state})
|
||||
|
||||
frame = await asyncio.wait_for(
|
||||
browser_sessions.screenshot(session_id), timeout=5.0)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("ws screenshot timeout for %s", session_id[:12])
|
||||
await asyncio.sleep(interval)
|
||||
continue
|
||||
except KeyError:
|
||||
await websocket.send_json({"error": "session_not_found"})
|
||||
break
|
||||
except Exception as exc:
|
||||
logger.warning("ws screenshot error: %s", exc)
|
||||
await asyncio.sleep(interval)
|
||||
continue
|
||||
|
||||
frame_hash = hashlib.md5(frame).hexdigest()
|
||||
if frame_hash != last_frame_hash:
|
||||
last_frame_hash = frame_hash
|
||||
unchanged_count = 0
|
||||
try:
|
||||
await websocket.send_bytes(frame)
|
||||
except Exception:
|
||||
break
|
||||
else:
|
||||
unchanged_count += 1
|
||||
|
||||
await asyncio.sleep(max(_WS_MIN_INTERVAL, interval))
|
||||
except (WebSocketDisconnect, asyncio.CancelledError):
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.debug("ws push_loop ended: %s", exc)
|
||||
|
||||
# Send initial metadata so client knows session info
|
||||
try:
|
||||
state = await browser_sessions.state(session_id)
|
||||
await websocket.send_json({"type": "init", "session": state})
|
||||
except Exception as exc:
|
||||
await websocket.send_json({"error": f"session error: {exc}"})
|
||||
await websocket.close()
|
||||
return
|
||||
|
||||
recv_task = asyncio.create_task(receive_loop())
|
||||
push_task = asyncio.create_task(push_loop())
|
||||
|
||||
# Run until one side closes
|
||||
done, pending = await asyncio.wait(
|
||||
[recv_task, push_task],
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
for t in pending:
|
||||
t.cancel()
|
||||
try:
|
||||
await t
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
@@ -1,7 +1,6 @@
|
||||
"""Custom pages CRUD router + authenticated iframe proxy."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, List, Literal, Optional
|
||||
@@ -18,12 +17,8 @@ from app.models.admin_user import AdminUser
|
||||
from app.models.custom_page import CustomPage
|
||||
from app.models.upstream import Upstream
|
||||
from app.services.upstream_client import _find_user_id
|
||||
from app.services.auth_capture_service import extract_all
|
||||
from app.services.browser_session_service import browser_sessions
|
||||
from app.utils.auth import decode_token, get_current_user, get_user_from_token_param
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/custom-pages", tags=["custom-pages"])
|
||||
|
||||
# Headers that prevent iframe embedding — strip them from proxied responses
|
||||
@@ -45,7 +40,7 @@ class CustomPageCreate(BaseModel):
|
||||
sort_order: int = 0
|
||||
enabled: bool = True
|
||||
use_proxy: bool = False
|
||||
access_mode: Literal["direct", "proxy", "remote_browser"] = "direct"
|
||||
access_mode: Literal["direct", "proxy"] = "direct"
|
||||
description: Optional[str] = None
|
||||
login_username: Optional[str] = None
|
||||
login_password: Optional[str] = None
|
||||
@@ -63,7 +58,7 @@ class CustomPageUpdate(BaseModel):
|
||||
sort_order: Optional[int] = None
|
||||
enabled: Optional[bool] = None
|
||||
use_proxy: Optional[bool] = None
|
||||
access_mode: Optional[Literal["direct", "proxy", "remote_browser"]] = None
|
||||
access_mode: Optional[Literal["direct", "proxy"]] = None
|
||||
description: Optional[str] = None
|
||||
login_username: Optional[str] = None
|
||||
login_password: Optional[str] = None
|
||||
@@ -118,7 +113,7 @@ def _page_response(page: CustomPage) -> CustomPageResponse:
|
||||
sort_order=page.sort_order,
|
||||
enabled=page.enabled,
|
||||
use_proxy=page.use_proxy,
|
||||
access_mode=page.access_mode,
|
||||
access_mode="proxy" if page.use_proxy or page.access_mode == "proxy" else "direct",
|
||||
description=page.description,
|
||||
login_username=page.login_username,
|
||||
login_username_selector=page.login_username_selector,
|
||||
@@ -143,6 +138,8 @@ def list_pages(db: Session = Depends(get_db), _=Depends(get_current_user)):
|
||||
@router.post("", response_model=CustomPageResponse, status_code=201)
|
||||
def create_page(body: CustomPageCreate, db: Session = Depends(get_db), _=Depends(get_current_user)):
|
||||
data = body.model_dump()
|
||||
if "access_mode" not in body.model_fields_set and data.get("use_proxy"):
|
||||
data["access_mode"] = "proxy"
|
||||
data["use_proxy"] = data["access_mode"] == "proxy"
|
||||
for key in (
|
||||
"login_username",
|
||||
@@ -210,166 +207,6 @@ def delete_page(pid: int, db: Session = Depends(get_db), _=Depends(get_current_u
|
||||
db.commit()
|
||||
|
||||
|
||||
# ---- One-click refresh auth ----
|
||||
|
||||
import json as _json
|
||||
|
||||
|
||||
class RefreshAuthResponse(BaseModel):
|
||||
success: bool
|
||||
message: str
|
||||
warning: Optional[str] = None
|
||||
|
||||
|
||||
def _norm_path(value: Any) -> str:
|
||||
return str(value or "").strip().rstrip("/")
|
||||
|
||||
|
||||
def _detect_upstream_platform(upstream: Upstream, auth_config: dict) -> str:
|
||||
api_prefix = _norm_path(upstream.api_prefix)
|
||||
groups_endpoint = _norm_path(upstream.groups_endpoint)
|
||||
rate_endpoint = _norm_path(upstream.rate_endpoint)
|
||||
login_path = _norm_path(auth_config.get("login_path"))
|
||||
|
||||
if groups_endpoint == "/api/user/self/groups" or login_path == "/api/user/login":
|
||||
return "new-api-user"
|
||||
if api_prefix == "/api/v1" or groups_endpoint in {"/groups/available", "/groups/rates"} or login_path == "/auth/login":
|
||||
return "sub2api"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _first_candidate(candidates: list[dict], *types: str) -> Optional[dict]:
|
||||
for c in candidates:
|
||||
if c.get("type") in types:
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def _pick_best_candidate(candidates: list[dict], preferred_auth_type: str, platform: str = "unknown") -> Optional[dict]:
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
if platform == "sub2api":
|
||||
return _first_candidate(candidates, "bearer_token", "api_key")
|
||||
if platform == "new-api-user":
|
||||
return _first_candidate(candidates, "cookie_bundle", "cookie", "bearer_token", "api_key")
|
||||
if preferred_auth_type == "cookie":
|
||||
return _first_candidate(candidates, "cookie_bundle", "cookie")
|
||||
elif preferred_auth_type in ("bearer", "api_key"):
|
||||
type_map = {"bearer": "bearer_token", "api_key": "api_key"}
|
||||
preferred = type_map.get(preferred_auth_type)
|
||||
if preferred:
|
||||
return _first_candidate(candidates, preferred)
|
||||
# fallback：排序后取第一个
|
||||
return candidates[0]
|
||||
|
||||
|
||||
@router.post("/{pid}/refresh-auth", response_model=RefreshAuthResponse)
|
||||
async def refresh_auth(pid: int, db: Session = Depends(get_db), _=Depends(get_current_user)):
|
||||
page = db.query(CustomPage).filter(CustomPage.id == pid).first()
|
||||
if not page:
|
||||
raise HTTPException(404, "page not found")
|
||||
if page.access_mode != "remote_browser":
|
||||
raise HTTPException(400, "page is not in remote_browser mode")
|
||||
if not page.linked_upstream_id:
|
||||
raise HTTPException(400, "page has no linked upstream")
|
||||
upstream = db.query(Upstream).filter(Upstream.id == page.linked_upstream_id).first()
|
||||
if not upstream:
|
||||
raise HTTPException(404, "linked upstream not found")
|
||||
|
||||
try:
|
||||
session = browser_sessions.find_by_page_id(page.id)
|
||||
except KeyError:
|
||||
return RefreshAuthResponse(success=False, message="请先打开远程浏览器并登录")
|
||||
|
||||
try:
|
||||
result = await extract_all(session)
|
||||
except Exception as exc:
|
||||
return RefreshAuthResponse(success=False, message=f"提取失败: {exc}")
|
||||
|
||||
candidates = result.get("candidates", [])
|
||||
existing_config = _json.loads(upstream.auth_config_json or "{}")
|
||||
platform = _detect_upstream_platform(upstream, existing_config)
|
||||
candidate = _pick_best_candidate(candidates, upstream.auth_type, platform)
|
||||
if not candidate:
|
||||
if platform == "sub2api" and _first_candidate(candidates, "cookie_bundle", "cookie"):
|
||||
return RefreshAuthResponse(
|
||||
success=False,
|
||||
message="Sub2API 需要 Bearer Token；当前只提取到 Cookie。请在远程浏览器完成登录后刷新页面或触发一次接口请求，再重新提取。",
|
||||
)
|
||||
return RefreshAuthResponse(success=False, message="未提取到有效凭证，请确认已在远程浏览器中登录")
|
||||
|
||||
ctype = candidate["type"]
|
||||
|
||||
if ctype in ("cookie_bundle", "cookie"):
|
||||
upstream.auth_type = "cookie"
|
||||
# cookie_bundle.value 已是完整 cookie_string；cookie.value 是 "name=value" 格式
|
||||
existing_config["cookie_string"] = candidate.get("value", "")
|
||||
if candidate.get("new_api_user"):
|
||||
existing_config["new_api_user"] = candidate["new_api_user"]
|
||||
if platform == "new-api-user":
|
||||
upstream.api_prefix = ""
|
||||
upstream.groups_endpoint = "/api/user/self/groups"
|
||||
upstream.rate_endpoint = "/api/user/self/groups"
|
||||
elif ctype == "bearer_token":
|
||||
upstream.auth_type = "bearer"
|
||||
raw = candidate.get("value", "")
|
||||
# Clean up: strip whitespace, remove "Bearer " prefix if present
|
||||
token = raw.strip()
|
||||
if token.startswith("Bearer "):
|
||||
token = token[7:].strip()
|
||||
# Validate token can be used as HTTP header value
|
||||
try:
|
||||
token.encode("latin-1")
|
||||
except UnicodeEncodeError:
|
||||
return RefreshAuthResponse(
|
||||
success=False,
|
||||
message="提取到的 Token 含有非 HTTP 标头字符，请确认已在远程浏览器中正确登录并重试",
|
||||
)
|
||||
existing_config["token"] = token
|
||||
elif ctype == "api_key":
|
||||
upstream.auth_type = "api_key"
|
||||
existing_config["key"] = candidate.get("value", "")
|
||||
existing_config.setdefault("header", "X-API-Key")
|
||||
|
||||
upstream.auth_config_json = _json.dumps(existing_config, ensure_ascii=False)
|
||||
upstream.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
# ── 宽松验证：写回后尝试调用 get_available_groups 验证凭证可用性 ──
|
||||
# 失败时仍然 commit（凭证已写入），但在 message 里说明验证失败
|
||||
# 这样用户仍能看到新凭证已写入，便于 debug（cf_clearance 绑 IP 时验证必然失败）
|
||||
warning_msg: Optional[str] = None
|
||||
try:
|
||||
from app.services.upstream_client import UpstreamClient
|
||||
groups_endpoint = upstream.groups_endpoint or "/groups/available"
|
||||
new_auth_config = _json.loads(upstream.auth_config_json)
|
||||
with UpstreamClient(
|
||||
base_url=upstream.base_url,
|
||||
api_prefix=upstream.api_prefix or "",
|
||||
auth_type=upstream.auth_type,
|
||||
auth_config=new_auth_config,
|
||||
timeout=float(upstream.timeout_seconds or 30),
|
||||
) as uc:
|
||||
uc.get_available_groups(groups_endpoint)
|
||||
logger.info("refresh_auth: upstream %s credential verification passed", upstream.id)
|
||||
except Exception as exc:
|
||||
warning_msg = (
|
||||
f"凭证已写入但 API 验证失败：{exc}。"
|
||||
"若 SmartUp 与远程浏览器不在同一 IP，cf_clearance 可能无法复用，请手动测试连接。"
|
||||
)
|
||||
logger.warning(
|
||||
"refresh_auth: upstream %s credential verification failed (written anyway): %s",
|
||||
upstream.id, exc,
|
||||
)
|
||||
|
||||
db.commit()
|
||||
auth_type_label = upstream.auth_type
|
||||
cookie_count = candidate.get("cookie_count", "")
|
||||
count_str = f"（{cookie_count} 个 cookie）" if cookie_count else ""
|
||||
success_msg = f"凭证已刷新 ({auth_type_label}{count_str})"
|
||||
return RefreshAuthResponse(success=True, message=success_msg, warning=warning_msg)
|
||||
|
||||
|
||||
# ---- Frame Proxy (simple: strip X-Frame-Options / CSP, pass through content) ----
|
||||
|
||||
_STRIP_RESP = {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Auth credential extraction from remote browser sessions."""
|
||||
"""Credential candidate curation for real-browser auth imports."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
@@ -23,116 +23,6 @@ SESSION_COOKIE_NAMES = frozenset({
|
||||
})
|
||||
|
||||
|
||||
async def extract_cookies(session: Any) -> list[dict[str, Any]]:
|
||||
"""Extract all cookies from the browser context."""
|
||||
cookies = await session.context.cookies()
|
||||
return [
|
||||
{
|
||||
"name": c.get("name", ""),
|
||||
"value": c.get("value", ""),
|
||||
"domain": c.get("domain", ""),
|
||||
"httpOnly": c.get("httpOnly", False),
|
||||
"secure": c.get("secure", False),
|
||||
}
|
||||
for c in cookies
|
||||
]
|
||||
|
||||
|
||||
async def extract_local_storage(page: Any) -> dict[str, str]:
|
||||
try:
|
||||
raw = await page.evaluate("() => JSON.stringify(window.localStorage)")
|
||||
if isinstance(raw, str):
|
||||
return json.loads(raw)
|
||||
return raw or {}
|
||||
except Exception as exc:
|
||||
logger.debug("localStorage extraction failed: %s", exc)
|
||||
return {}
|
||||
|
||||
|
||||
async def extract_session_storage(page: Any) -> dict[str, str]:
|
||||
try:
|
||||
raw = await page.evaluate("() => JSON.stringify(window.sessionStorage)")
|
||||
if isinstance(raw, str):
|
||||
return json.loads(raw)
|
||||
return raw or {}
|
||||
except Exception as exc:
|
||||
logger.debug("sessionStorage extraction failed: %s", exc)
|
||||
return {}
|
||||
|
||||
|
||||
async def extract_new_api_user_id(page: Any) -> str:
|
||||
try:
|
||||
value = await page.evaluate("""
|
||||
async () => {
|
||||
const uid = localStorage.getItem('uid')
|
||||
if (uid) return uid
|
||||
const userRaw = localStorage.getItem('user')
|
||||
if (userRaw) {
|
||||
try {
|
||||
const user = JSON.parse(userRaw)
|
||||
if (user?.id) return String(user.id)
|
||||
} catch {}
|
||||
}
|
||||
const response = await fetch('/api/user/self', { credentials: 'include' })
|
||||
if (!response.ok) return ''
|
||||
const payload = await response.json()
|
||||
const data = payload?.data || payload
|
||||
return data?.id ? String(data.id) : ''
|
||||
}
|
||||
""")
|
||||
return str(value or "").strip()
|
||||
except Exception as exc:
|
||||
logger.debug("New-API user id extraction failed: %s", exc)
|
||||
return ""
|
||||
|
||||
|
||||
async def extract_request_headers(session: Any) -> list[dict[str, str]]:
|
||||
"""Return Authorization / API-Key headers captured continuously by CDP.
|
||||
|
||||
The CDP Network listener is started when the ephemeral session is created
|
||||
(in BrowserSessionService.create_ephemeral), so headers from the login
|
||||
flow are captured in real-time without needing a fresh CDP attach.
|
||||
"""
|
||||
if hasattr(session, "captured_headers") and session.captured_headers:
|
||||
logger.debug("auth-capture: returning %d cached headers", len(session.captured_headers))
|
||||
return list(session.captured_headers)
|
||||
return []
|
||||
|
||||
|
||||
async def extract_all(session: Any) -> dict[str, Any]:
|
||||
"""Extract all auth credentials from a browser session.
|
||||
|
||||
Returns:
|
||||
cookies, storage, session_storage, auth_headers, candidates
|
||||
"""
|
||||
page = session.page
|
||||
cookies = await extract_cookies(session)
|
||||
local_storage = await extract_local_storage(page)
|
||||
session_storage = await extract_session_storage(page)
|
||||
auth_headers = await extract_request_headers(session)
|
||||
new_api_user = _find_new_api_user(local_storage, session_storage) or await extract_new_api_user_id(page)
|
||||
|
||||
# 获取当前浏览器页面的真实 URL（比 session.url 更准确）
|
||||
page_url = ""
|
||||
try:
|
||||
page_url = page.url or ""
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
candidates = _curate_candidates(
|
||||
cookies, local_storage, session_storage, auth_headers, new_api_user,
|
||||
page_url=page_url,
|
||||
)
|
||||
|
||||
return {
|
||||
"cookies": cookies,
|
||||
"storage": local_storage,
|
||||
"session_storage": session_storage,
|
||||
"auth_headers": auth_headers,
|
||||
"candidates": candidates,
|
||||
}
|
||||
|
||||
|
||||
def _cookie_matches_hostname(cookie_domain: str, hostname: str) -> bool:
|
||||
"""判断 cookie domain 是否适用于给定 hostname。
|
||||
|
||||
|
||||
@@ -1,951 +0,0 @@
|
||||
"""Managed Playwright browser sessions for custom pages."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import urlparse
|
||||
from uuid import uuid4
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BrowserDependencyError(RuntimeError):
|
||||
"""Raised when Playwright or its browser runtime is unavailable."""
|
||||
|
||||
|
||||
class BrowserSessionError(RuntimeError):
|
||||
"""Raised when an existing browser session can no longer be used."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class BrowserTab:
|
||||
id: str
|
||||
page: Any
|
||||
created_at: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class BrowserSession:
|
||||
id: str
|
||||
custom_page_id: int
|
||||
profile_key: str
|
||||
context: Any
|
||||
tabs: dict[str, BrowserTab]
|
||||
active_tab_id: str
|
||||
lock: asyncio.Lock
|
||||
tab_revision: int = 0
|
||||
cdp_session: Any = None
|
||||
captured_headers: list[dict] = None # auth headers from CDP
|
||||
last_saved_state_at: float = 0.0
|
||||
|
||||
@property
|
||||
def active_tab(self) -> BrowserTab:
|
||||
return self.tabs[self.active_tab_id]
|
||||
|
||||
@property
|
||||
def page(self) -> Any:
|
||||
return self.active_tab.page
|
||||
|
||||
|
||||
class BrowserSessionService:
|
||||
# Idle TTL: close sessions that haven't had activity for this long
|
||||
IDLE_TTL_SECONDS = 1800 # 30 minutes
|
||||
# Cap: max concurrent persistent sessions (excludes auth-capture)
|
||||
MAX_SESSIONS = 10
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._playwright: Optional[Any] = None
|
||||
self._sessions: dict[str, BrowserSession] = {}
|
||||
self._profiles: dict[str, str] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
self._last_event_at: dict[str, float] = {}
|
||||
self._evict_task: Optional[asyncio.Task[None]] = None
|
||||
|
||||
def _browser_launch_kwargs(self, width: int, height: int) -> dict[str, Any]:
|
||||
return {
|
||||
"headless": get_settings().browser_headless,
|
||||
"viewport": {"width": width, "height": height},
|
||||
"color_scheme": "dark",
|
||||
"locale": "zh-CN",
|
||||
"timezone_id": get_settings().tz,
|
||||
"ignore_default_args": ["--enable-automation"],
|
||||
"args": [
|
||||
"--no-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--window-size=%d,%d" % (width, height),
|
||||
],
|
||||
}
|
||||
|
||||
async def _install_browser_init_scripts(self, context: Any) -> None:
|
||||
await context.add_init_script("""
|
||||
(() => {
|
||||
try {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
Object.defineProperty(navigator, 'languages', { get: () => ['zh-CN', 'zh', 'en-US', 'en'] });
|
||||
Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
|
||||
window.chrome = window.chrome || { runtime: {} };
|
||||
} catch (_) {}
|
||||
})();
|
||||
""")
|
||||
|
||||
async def create(
|
||||
self,
|
||||
custom_page_id: int,
|
||||
url: str,
|
||||
width: int = 1280,
|
||||
height: int = 720,
|
||||
login_config: Optional[dict[str, Any]] = None,
|
||||
) -> BrowserSession:
|
||||
if not url.startswith(("http://", "https://")):
|
||||
raise ValueError("Only http/https URLs are allowed")
|
||||
width = max(320, min(width, 2560))
|
||||
height = max(240, min(height, 1600))
|
||||
async with self._lock:
|
||||
await self._ensure_playwright()
|
||||
profile_key = self._profile_key(custom_page_id, url)
|
||||
existing_id = self._profiles.get(profile_key)
|
||||
existing = self._sessions.get(existing_id or "")
|
||||
if existing and not existing.page.is_closed():
|
||||
# Health check: verify session can actually serve content
|
||||
healthy = True
|
||||
try:
|
||||
async with existing.lock:
|
||||
url_before = existing.page.url
|
||||
await existing.page.evaluate("1") # ping
|
||||
await existing.page.screenshot(type="jpeg", quality=10, timeout=5000)
|
||||
await existing.page.set_viewport_size({"width": width, "height": height})
|
||||
if url_before == "about:blank":
|
||||
await existing.page.goto(url, wait_until="domcontentloaded", timeout=45000)
|
||||
await self._autofill_login(existing.page, login_config)
|
||||
await self._reset_page_zoom(existing)
|
||||
self._touch(existing.id)
|
||||
except Exception:
|
||||
logger.info("existing session %s unhealthy, recreating", existing.id[:12])
|
||||
healthy = False
|
||||
if healthy:
|
||||
return existing
|
||||
# Close unhealthy session (profile stays on disk)
|
||||
await self.close(existing.id)
|
||||
if existing_id:
|
||||
self._profiles.pop(profile_key, None)
|
||||
# Idle cleanup: close stale sessions before spawning new ones
|
||||
await self._evict_idle_sessions()
|
||||
|
||||
context = await self._playwright.chromium.launch_persistent_context(
|
||||
str(self._profile_dir(profile_key)),
|
||||
**self._browser_launch_kwargs(width, height),
|
||||
)
|
||||
await self._install_browser_init_scripts(context)
|
||||
await self._restore_session_state(context, profile_key)
|
||||
# Grant clipboard access for the page origin
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
origin = f"{parsed.scheme}://{parsed.netloc}"
|
||||
await context.grant_permissions(["clipboard-read", "clipboard-write"], origin=origin)
|
||||
except Exception:
|
||||
logger.debug("clipboard permission grant failed (non-fatal)")
|
||||
page = context.pages[0] if context.pages else await context.new_page()
|
||||
tab_id = uuid4().hex
|
||||
tab = BrowserTab(id=tab_id, page=page, created_at=asyncio.get_event_loop().time())
|
||||
session = BrowserSession(
|
||||
id=uuid4().hex,
|
||||
custom_page_id=custom_page_id,
|
||||
profile_key=profile_key,
|
||||
context=context,
|
||||
tabs={tab_id: tab},
|
||||
active_tab_id=tab_id,
|
||||
lock=asyncio.Lock(),
|
||||
)
|
||||
self._sessions[session.id] = session
|
||||
self._profiles[profile_key] = session.id
|
||||
self._touch(session.id)
|
||||
# Register listeners for the initial tab
|
||||
self._setup_tab_listeners(session, page)
|
||||
# Register page capture for multi-tab support
|
||||
context.on("page", lambda p: self._handle_new_page(session, p))
|
||||
# Evict again after adding the new session so cap is enforced immediately
|
||||
await self._evict_idle_sessions()
|
||||
try:
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=45000)
|
||||
await self._autofill_login(page, login_config)
|
||||
await self._reset_page_zoom(session)
|
||||
except Exception:
|
||||
await self.close(session.id)
|
||||
raise
|
||||
logger.info("session created: %s (page=%s, profile=%s)", session.id[:12], custom_page_id, profile_key)
|
||||
return session
|
||||
|
||||
def _touch(self, session_id: str) -> None:
|
||||
"""Mark a session as recently active (reset idle timer)."""
|
||||
self._last_event_at[session_id] = asyncio.get_event_loop().time()
|
||||
|
||||
def _handle_new_page(self, session: BrowserSession, page: Any) -> None:
|
||||
"""Capture a new page opened by the remote browser (e.g. target="_blank")."""
|
||||
asyncio.create_task(self._register_new_page(session, page))
|
||||
|
||||
def _setup_tab_listeners(self, session: BrowserSession, page: Any) -> None:
|
||||
"""Register navigation and state listeners to bump tab_revision."""
|
||||
def bump_revision(_=None):
|
||||
session.tab_revision += 1
|
||||
|
||||
page.on("domcontentloaded", bump_revision)
|
||||
page.on("load", bump_revision)
|
||||
page.on("framenavigated", bump_revision)
|
||||
page.on("close", bump_revision)
|
||||
|
||||
async def _register_new_page(self, session: BrowserSession, page: Any) -> None:
|
||||
tab_id = uuid4().hex
|
||||
tab = BrowserTab(id=tab_id, page=page, created_at=asyncio.get_event_loop().time())
|
||||
|
||||
async with session.lock:
|
||||
session.tabs[tab_id] = tab
|
||||
session.active_tab_id = tab_id
|
||||
session.tab_revision += 1
|
||||
logger.info("session %s: captured new tab %s (total: %d)", session.id[:12], tab_id[:8], len(session.tabs))
|
||||
|
||||
self._setup_tab_listeners(session, page)
|
||||
# Best-effort: bring to front and reset zoom
|
||||
await self._init_new_tab(session, tab)
|
||||
|
||||
async def _init_new_tab(self, session: BrowserSession, tab: BrowserTab) -> None:
|
||||
try:
|
||||
await tab.page.bring_to_front()
|
||||
await self._reset_page_zoom(session)
|
||||
# Grant clipboard permission for the new page's origin if possible
|
||||
try:
|
||||
url = tab.page.url
|
||||
if url.startswith("http"):
|
||||
parsed = urlparse(url)
|
||||
origin = f"{parsed.scheme}://{parsed.netloc}"
|
||||
await session.context.grant_permissions(["clipboard-read", "clipboard-write"], origin=origin)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.debug("new tab %s init failed: %s", tab.id[:8], exc)
|
||||
|
||||
async def screenshot(self, session_id: str) -> bytes:
|
||||
session = self._get(session_id)
|
||||
self._touch(session_id)
|
||||
async with session.lock:
|
||||
self._ensure_open(session)
|
||||
if session.profile_key and not session.profile_key.startswith("auth-capture-"):
|
||||
now = time.monotonic()
|
||||
if now - session.last_saved_state_at > 10.0:
|
||||
await self._save_session_state(session)
|
||||
session.last_saved_state_at = now
|
||||
return await session.page.screenshot(type="jpeg", quality=65, full_page=False)
|
||||
|
||||
async def event(
|
||||
self,
|
||||
session_id: str,
|
||||
event_type: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
include_state: bool = True,
|
||||
) -> dict[str, Any] | None:
|
||||
session = self._get(session_id)
|
||||
self._last_event_at[session_id] = asyncio.get_event_loop().time()
|
||||
async with session.lock:
|
||||
self._ensure_open(session)
|
||||
page = session.page
|
||||
if event_type == "click":
|
||||
await page.mouse.click(float(payload["x"]), float(payload["y"]), button=payload.get("button", "left"))
|
||||
elif event_type == "dblclick":
|
||||
await page.mouse.dblclick(float(payload["x"]), float(payload["y"]), button=payload.get("button", "left"))
|
||||
elif event_type == "mousemove":
|
||||
await page.mouse.move(float(payload["x"]), float(payload["y"]))
|
||||
elif event_type == "mousedown":
|
||||
await page.mouse.move(float(payload["x"]), float(payload["y"]))
|
||||
await page.mouse.down(button=payload.get("button", "left"))
|
||||
elif event_type == "mouseup":
|
||||
await page.mouse.move(float(payload["x"]), float(payload["y"]))
|
||||
await page.mouse.up(button=payload.get("button", "left"))
|
||||
elif event_type == "type":
|
||||
text = str(payload.get("text", ""))
|
||||
if text:
|
||||
await page.keyboard.insert_text(text)
|
||||
elif event_type == "key":
|
||||
key = str(payload.get("key", ""))
|
||||
if key:
|
||||
await page.keyboard.press(key)
|
||||
elif event_type == "scroll":
|
||||
if payload.get("x") is not None and payload.get("y") is not None:
|
||||
await page.mouse.move(float(payload["x"]), float(payload["y"]))
|
||||
await page.mouse.wheel(float(payload.get("delta_x", 0)), float(payload.get("delta_y", 0)))
|
||||
elif event_type == "reload":
|
||||
await page.reload(wait_until="domcontentloaded", timeout=45000)
|
||||
elif event_type == "back":
|
||||
await page.go_back(wait_until="domcontentloaded", timeout=45000)
|
||||
elif event_type == "forward":
|
||||
await page.go_forward(wait_until="domcontentloaded", timeout=45000)
|
||||
elif event_type == "resize":
|
||||
width = max(320, min(int(payload.get("width", 1280)), 2560))
|
||||
height = max(240, min(int(payload.get("height", 720)), 1600))
|
||||
await page.set_viewport_size({"width": width, "height": height})
|
||||
else:
|
||||
raise ValueError("Unsupported browser event")
|
||||
if session.profile_key and not session.profile_key.startswith("auth-capture-"):
|
||||
now = time.monotonic()
|
||||
if now - session.last_saved_state_at > 5.0:
|
||||
await self._save_session_state(session)
|
||||
session.last_saved_state_at = now
|
||||
|
||||
if not include_state:
|
||||
return None
|
||||
return await self._session_state(session)
|
||||
|
||||
async def selected_text(self, session_id: str) -> str:
|
||||
session = self._get(session_id)
|
||||
self._touch(session_id)
|
||||
async with session.lock:
|
||||
self._ensure_open(session)
|
||||
value = await session.page.evaluate("() => window.getSelection()?.toString() || ''")
|
||||
return str(value or "")
|
||||
|
||||
async def read_clipboard(self, session_id: str) -> tuple[Optional[str], Optional[str]]:
|
||||
"""Read the remote browser's clipboard text.
|
||||
|
||||
Returns (text, error_reason).
|
||||
text is None when the clipboard is empty or unreadable.
|
||||
error_reason is None on success or "empty" — non-None indicates a genuine failure.
|
||||
"""
|
||||
session = self._get(session_id)
|
||||
self._touch(session_id)
|
||||
async with session.lock:
|
||||
self._ensure_open(session)
|
||||
try:
|
||||
result = await session.page.evaluate("""
|
||||
async () => {
|
||||
try {
|
||||
const text = await navigator.clipboard.readText();
|
||||
return text || null;
|
||||
} catch (e) {
|
||||
if (e instanceof DOMException) {
|
||||
if (e.name === 'NotAllowedError') return 'ERROR:denied';
|
||||
if (e.name === 'NotFoundError') return null;
|
||||
}
|
||||
return 'ERROR:' + (e.message || String(e));
|
||||
}
|
||||
}
|
||||
""")
|
||||
if result is None:
|
||||
return None, None # empty clipboard
|
||||
if isinstance(result, str) and result.startswith("ERROR:"):
|
||||
reason = result[6:]
|
||||
logger.debug("clipboard read error for %s: %s", session_id[:12], reason)
|
||||
return None, reason
|
||||
return str(result), None
|
||||
except Exception as exc:
|
||||
logger.warning("clipboard read failed for %s: %s", session_id[:12], exc)
|
||||
return None, "read_failed"
|
||||
|
||||
async def close(self, session_id: str) -> None:
|
||||
self._last_event_at.pop(session_id, None)
|
||||
session = self._discard_session(session_id)
|
||||
if not session:
|
||||
return
|
||||
logger.info("session closing: %s (page=%s, profile=%s)", session_id[:12], session.custom_page_id, session.profile_key)
|
||||
|
||||
# 在完全关闭 context 前，强制将最新的状态落盘保存
|
||||
if session.profile_key and not session.profile_key.startswith("auth-capture-"):
|
||||
try:
|
||||
if not session.page.is_closed():
|
||||
await self._save_session_state(session)
|
||||
except Exception as exc:
|
||||
logger.debug("failed to save state during close: %s", exc)
|
||||
|
||||
# Detach CDP session if active
|
||||
if session.cdp_session:
|
||||
try:
|
||||
await session.cdp_session.detach()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
close_ok = True
|
||||
# 关闭 context 带超时，避免永远挂起
|
||||
try:
|
||||
await asyncio.wait_for(session.context.close(), timeout=10.0)
|
||||
logger.info("session context closed: %s", session_id[:12])
|
||||
except asyncio.TimeoutError:
|
||||
close_ok = False
|
||||
logger.warning("session close timeout: %s (falling back to browser.close)", session_id[:12])
|
||||
try:
|
||||
browser = getattr(session.context, "browser", None)
|
||||
if browser is not None:
|
||||
await asyncio.wait_for(browser.close(), timeout=5.0)
|
||||
close_ok = True
|
||||
logger.info("session browser fallback closed: %s", session_id[:12])
|
||||
else:
|
||||
logger.warning("session context.browser is None, cannot fallback: %s", session_id[:12])
|
||||
except Exception as exc:
|
||||
logger.warning("session browser fallback failed: %s: %s", session_id[:12], exc)
|
||||
except Exception as exc:
|
||||
close_ok = False
|
||||
logger.warning("session close error: %s: %s", session_id[:12], exc)
|
||||
|
||||
# Clean up ephemeral (auth-capture) profile directories
|
||||
if session.profile_key and session.profile_key.startswith("auth-capture-"):
|
||||
profile_dir = self._profile_dir(session.profile_key)
|
||||
import shutil
|
||||
try:
|
||||
shutil.rmtree(profile_dir, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if close_ok:
|
||||
logger.info("session closed: %s", session_id[:12])
|
||||
else:
|
||||
logger.warning("session close_failed: %s", session_id[:12])
|
||||
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
# Cancel the background eviction loop
|
||||
if self._evict_task is not None and not self._evict_task.done():
|
||||
self._evict_task.cancel()
|
||||
try:
|
||||
await self._evict_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._evict_task = None
|
||||
sessions = list(self._sessions)
|
||||
if sessions:
|
||||
logger.info("shutdown: closing %d browser sessions", len(sessions))
|
||||
for session_id in sessions:
|
||||
try:
|
||||
await asyncio.wait_for(self.close(session_id), timeout=15.0)
|
||||
except Exception as exc:
|
||||
logger.warning("shutdown close failed for %s: %s", session_id[:12], exc)
|
||||
if self._playwright:
|
||||
logger.info("shutdown: stopping playwright")
|
||||
try:
|
||||
await asyncio.wait_for(self._playwright.stop(), timeout=10.0)
|
||||
except Exception as exc:
|
||||
logger.warning("shutdown playwright stop failed: %s", exc)
|
||||
self._playwright = None
|
||||
|
||||
async def state(self, session_id: str) -> dict[str, Any]:
|
||||
session = self._get(session_id)
|
||||
self._touch(session_id)
|
||||
async with session.lock:
|
||||
self._ensure_open(session)
|
||||
return await self._session_state(session)
|
||||
|
||||
async def activate_tab(self, session_id: str, tab_id: str) -> dict[str, Any]:
|
||||
session = self._get(session_id)
|
||||
self._touch(session_id)
|
||||
async with session.lock:
|
||||
self._ensure_open(session)
|
||||
if tab_id not in session.tabs:
|
||||
raise KeyError("tab not found")
|
||||
session.active_tab_id = tab_id
|
||||
session.tab_revision += 1
|
||||
await session.page.bring_to_front()
|
||||
return await self._session_state(session)
|
||||
|
||||
async def close_tab(self, session_id: str, tab_id: str) -> dict[str, Any]:
|
||||
session = self._get(session_id)
|
||||
self._touch(session_id)
|
||||
async with session.lock:
|
||||
self._ensure_open(session)
|
||||
if tab_id not in session.tabs:
|
||||
raise KeyError("tab not found")
|
||||
if len(session.tabs) <= 1:
|
||||
raise ValueError("cannot close the last tab")
|
||||
|
||||
tab = session.tabs.pop(tab_id)
|
||||
try:
|
||||
await tab.page.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if session.active_tab_id == tab_id:
|
||||
# Pick the latest remaining tab
|
||||
latest = max(session.tabs.values(), key=lambda t: t.created_at)
|
||||
session.active_tab_id = latest.id
|
||||
await session.page.bring_to_front()
|
||||
|
||||
session.tab_revision += 1
|
||||
return await self._session_state(session)
|
||||
|
||||
async def _session_state(self, session: BrowserSession) -> dict[str, Any]:
|
||||
tabs = []
|
||||
# We might need to prune closed pages during state generation too
|
||||
closed_ids = []
|
||||
# Use list() to avoid RuntimeError if tabs dict changes during iteration
|
||||
for tid, tab in list(session.tabs.items()):
|
||||
if tab.page.is_closed():
|
||||
closed_ids.append(tid)
|
||||
continue
|
||||
try:
|
||||
title = await tab.page.title()
|
||||
url = tab.page.url
|
||||
except Exception:
|
||||
title, url = "Loading...", "about:blank"
|
||||
tabs.append({
|
||||
"id": tid,
|
||||
"title": title,
|
||||
"url": url,
|
||||
"created_at": tab.created_at,
|
||||
})
|
||||
|
||||
if closed_ids:
|
||||
for cid in closed_ids:
|
||||
session.tabs.pop(cid, None)
|
||||
if not session.tabs:
|
||||
raise BrowserSessionError("all browser pages are closed")
|
||||
if session.active_tab_id in closed_ids:
|
||||
latest = max(session.tabs.values(), key=lambda t: t.created_at)
|
||||
session.active_tab_id = latest.id
|
||||
session.tab_revision += 1
|
||||
|
||||
tabs.sort(key=lambda x: x["created_at"])
|
||||
return {
|
||||
"id": session.id,
|
||||
"custom_page_id": session.custom_page_id,
|
||||
"url": session.page.url,
|
||||
"title": await session.page.title(),
|
||||
"active_tab_id": session.active_tab_id,
|
||||
"tabs": tabs,
|
||||
"tab_revision": session.tab_revision,
|
||||
}
|
||||
|
||||
async def _ensure_playwright(self) -> None:
|
||||
if self._playwright:
|
||||
return
|
||||
try:
|
||||
from playwright.async_api import async_playwright
|
||||
except ImportError as exc:
|
||||
raise BrowserDependencyError("Playwright is not installed. Run `pip install -r requirements.txt`.") from exc
|
||||
try:
|
||||
self._playwright = await async_playwright().start()
|
||||
except Exception as exc:
|
||||
raise BrowserDependencyError(f"Unable to start Playwright: {exc}") from exc
|
||||
# Start background eviction loop
|
||||
if self._evict_task is None or self._evict_task.done():
|
||||
self._evict_task = asyncio.create_task(self._evict_loop())
|
||||
|
||||
async def _reset_page_zoom(self, session: BrowserSession) -> None:
|
||||
try:
|
||||
cdp = await session.context.new_cdp_session(session.page)
|
||||
try:
|
||||
await cdp.send("Emulation.setPageScaleFactor", {"pageScaleFactor": 1})
|
||||
finally:
|
||||
await cdp.detach()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def autofill_login(
|
||||
self,
|
||||
session_id: str,
|
||||
login_config: Optional[dict[str, Any]],
|
||||
) -> bool:
|
||||
"""Public: manually trigger login autofill for an active session.
|
||||
|
||||
Only fills username/password fields — never auto-submits.
|
||||
Returns True if fields were found and filled, False otherwise.
|
||||
Never returns password data to the caller.
|
||||
"""
|
||||
session = self._get(session_id)
|
||||
self._touch(session_id)
|
||||
async with session.lock:
|
||||
self._ensure_open(session)
|
||||
return await self._autofill_login(session.page, login_config, max_wait_seconds=3.0, skip_submit=True)
|
||||
|
||||
async def _autofill_login(
|
||||
self,
|
||||
page: Any,
|
||||
config: Optional[dict[str, Any]],
|
||||
*,
|
||||
max_wait_seconds: float = 2.0,
|
||||
poll_interval_seconds: float = 0.25,
|
||||
skip_submit: bool = False,
|
||||
) -> bool:
|
||||
if not config or not config.get("enabled"):
|
||||
return False
|
||||
username = str(config.get("username") or "")
|
||||
password = str(config.get("password") or "")
|
||||
if not username or not password:
|
||||
return False
|
||||
try:
|
||||
username_selectors = [
|
||||
config.get("username_selector"),
|
||||
"input[type='email']",
|
||||
"input[name*='user' i]",
|
||||
"input[id*='user' i]",
|
||||
"input[name*='email' i]",
|
||||
"input[id*='email' i]",
|
||||
"input[name*='login' i]",
|
||||
"input[id*='login' i]",
|
||||
"input[autocomplete='username']",
|
||||
"input:not([type]), input[type='text']",
|
||||
]
|
||||
password_selectors = [
|
||||
config.get("password_selector"),
|
||||
"input[type='password']",
|
||||
"input[autocomplete='current-password']",
|
||||
]
|
||||
username_locator, password_locator = await self._wait_for_login_locators(
|
||||
page,
|
||||
username_selectors,
|
||||
password_selectors,
|
||||
max_wait_seconds=max_wait_seconds,
|
||||
poll_interval_seconds=poll_interval_seconds,
|
||||
)
|
||||
if not username_locator or not password_locator:
|
||||
logger.info("Login autofill skipped: login fields not found")
|
||||
return False
|
||||
await username_locator.fill(username, timeout=3000)
|
||||
await password_locator.fill(password, timeout=3000)
|
||||
if not skip_submit:
|
||||
submit_selector = str(config.get("submit_selector") or "").strip()
|
||||
if submit_selector:
|
||||
submit = await self._first_visible_locator(page, [submit_selector], timeout=500)
|
||||
if submit:
|
||||
await submit.click(timeout=3000)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.info("Login autofill skipped: %s", exc)
|
||||
return False
|
||||
|
||||
async def _wait_for_login_locators(
|
||||
self,
|
||||
page: Any,
|
||||
username_selectors: list[Optional[str]],
|
||||
password_selectors: list[Optional[str]],
|
||||
*,
|
||||
max_wait_seconds: float,
|
||||
poll_interval_seconds: float,
|
||||
) -> tuple[Optional[Any], Optional[Any]]:
|
||||
deadline = time.monotonic() + max_wait_seconds
|
||||
while True:
|
||||
username_locator = await self._first_visible_locator(page, username_selectors, timeout=150)
|
||||
password_locator = await self._first_visible_locator(page, password_selectors, timeout=150)
|
||||
if username_locator and password_locator:
|
||||
return username_locator, password_locator
|
||||
if time.monotonic() >= deadline:
|
||||
return None, None
|
||||
await asyncio.sleep(poll_interval_seconds)
|
||||
|
||||
async def _first_visible_locator(
|
||||
self,
|
||||
page: Any,
|
||||
selectors: list[Optional[str]],
|
||||
*,
|
||||
timeout: float = 1500,
|
||||
) -> Optional[Any]:
|
||||
for selector in selectors:
|
||||
selector = str(selector or "").strip()
|
||||
if not selector:
|
||||
continue
|
||||
try:
|
||||
locator = page.locator(selector).first
|
||||
if await locator.count() and await locator.is_visible(timeout=timeout):
|
||||
return locator
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
def get_session(self, session_id: str) -> BrowserSession:
|
||||
"""Retrieve a session by id — raises KeyError if missing."""
|
||||
session = self._sessions.get(session_id)
|
||||
if not session:
|
||||
raise KeyError("browser session not found")
|
||||
return session
|
||||
|
||||
def find_by_page_id(self, custom_page_id: int) -> BrowserSession:
|
||||
"""Find the active session for a custom page. Raises KeyError if none."""
|
||||
for session in self._sessions.values():
|
||||
if session.custom_page_id == custom_page_id and not session.page.is_closed():
|
||||
return session
|
||||
raise KeyError(f"no active browser session for page {custom_page_id}")
|
||||
|
||||
_get = get_session # alias for internal use
|
||||
|
||||
def _ensure_open(self, session: BrowserSession) -> None:
|
||||
if session.active_tab.page.is_closed():
|
||||
# Current tab closed? Try to cleanup and find another one
|
||||
session.tabs.pop(session.active_tab_id, None)
|
||||
if session.tabs:
|
||||
# Pick the latest created tab
|
||||
latest = max(session.tabs.values(), key=lambda t: t.created_at)
|
||||
session.active_tab_id = latest.id
|
||||
session.tab_revision += 1
|
||||
logger.info("active tab closed, switched to %s", latest.id[:8])
|
||||
else:
|
||||
self._discard_session(session.id)
|
||||
raise BrowserSessionError("all browser pages are closed")
|
||||
|
||||
def _discard_session(self, session_id: str) -> BrowserSession | None:
|
||||
session = self._sessions.pop(session_id, None)
|
||||
if session and self._profiles.get(session.profile_key) == session_id:
|
||||
self._profiles.pop(session.profile_key, None)
|
||||
return session
|
||||
|
||||
async def _evict_loop(self) -> None:
|
||||
"""Background loop that runs every 5 minutes to evict idle sessions."""
|
||||
while True:
|
||||
await asyncio.sleep(300) # 5 minutes
|
||||
try:
|
||||
await self._evict_idle_sessions()
|
||||
except Exception:
|
||||
logger.exception("idle eviction loop error")
|
||||
|
||||
async def _evict_idle_sessions(self) -> None:
|
||||
"""Close oldest idle sessions when over cap, or any past TTL.
|
||||
|
||||
- Auth-capture sessions: max 10 minutes lifetime.
|
||||
- Remote browser sessions: close after IDLE_TTL_SECONDS of no WebSocket activity.
|
||||
"""
|
||||
now = asyncio.get_event_loop().time()
|
||||
to_remove: list[str] = []
|
||||
for sid, session in self._sessions.items():
|
||||
if session.profile_key and session.profile_key.startswith("auth-capture-"):
|
||||
# auth-capture: max 10 minute TTL from creation
|
||||
created = session.tabs.get(session.active_tab_id)
|
||||
if created:
|
||||
age = now - created.created_at
|
||||
if age > 600:
|
||||
to_remove.append(sid)
|
||||
logger.info("evicting auth-capture session %s (age=%ds > 600s)", sid[:12], int(age))
|
||||
else:
|
||||
# remote browser sessions: idle TTL
|
||||
last_active = self._last_event_at.get(sid, 0.0)
|
||||
if last_active > 0 and (now - last_active) > self.IDLE_TTL_SECONDS:
|
||||
to_remove.append(sid)
|
||||
logger.info("evicting idle session %s (no activity for >%ds)", sid[:12], self.IDLE_TTL_SECONDS)
|
||||
for sid in to_remove:
|
||||
await self.close(sid)
|
||||
|
||||
# Second: if still over cap, evict oldest by last_event_at
|
||||
persistent = [(sid, s) for sid, s in self._sessions.items()
|
||||
if not (s.profile_key or "").startswith("auth-capture-")]
|
||||
if len(persistent) > self.MAX_SESSIONS:
|
||||
persistent.sort(key=lambda x: self._last_event_at.get(x[0], 0.0))
|
||||
excess = len(persistent) - self.MAX_SESSIONS
|
||||
for sid, _ in persistent[:excess]:
|
||||
logger.info("evicting session %s (over cap of %d)", sid[:12], self.MAX_SESSIONS)
|
||||
await self.close(sid)
|
||||
|
||||
async def clear_profile(self, custom_page_id: int, url: str) -> None:
|
||||
"""Close session for the page if active, then delete profile directory.
|
||||
|
||||
Raises RuntimeError if the directory cannot be fully removed.
|
||||
"""
|
||||
import shutil
|
||||
# Close active session and use its profile_key (precise match)
|
||||
profile_key: Optional[str] = None
|
||||
try:
|
||||
session = self.find_by_page_id(custom_page_id)
|
||||
profile_key = session.profile_key
|
||||
await self.close(session.id)
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
# Fallback: compute from URL (may be wrong if URL changed since session was created)
|
||||
if not profile_key:
|
||||
profile_key = self._profile_key(custom_page_id, url)
|
||||
|
||||
profile_dir = self._profile_dir(profile_key)
|
||||
if profile_dir.exists():
|
||||
shutil.rmtree(profile_dir) # no ignore_errors — let failure surface
|
||||
if profile_dir.exists():
|
||||
raise RuntimeError(
|
||||
f"Failed to fully remove browser profile directory: {profile_dir}"
|
||||
)
|
||||
logger.info("cleared browser profile for page %d: %s", custom_page_id, profile_dir)
|
||||
|
||||
def _profile_dir(self, profile_key: str) -> Path:
|
||||
root = Path(get_settings().browser_profiles_dir)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
profile = root / profile_key
|
||||
profile.mkdir(parents=True, exist_ok=True)
|
||||
return profile
|
||||
|
||||
def _cookies_path(self, profile_key: str) -> Path:
|
||||
return self._profile_dir(profile_key) / "session-cookies.json"
|
||||
|
||||
def _profile_key(self, custom_page_id: int, url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
origin = f"{parsed.scheme}-{parsed.netloc}".lower()
|
||||
safe_origin = re.sub(r"[^a-z0-9_.-]+", "_", origin).strip("_") or "page"
|
||||
return f"page-{custom_page_id}-{safe_origin[:80]}"
|
||||
|
||||
async def create_ephemeral(
|
||||
self,
|
||||
url: str,
|
||||
width: int = 1280,
|
||||
height: int = 720,
|
||||
) -> BrowserSession:
|
||||
"""Create a temporary browser session without a CustomPage record.
|
||||
|
||||
The session uses an isolated random-named profile so it never collides
|
||||
with persistent custom-page profiles. Caller MUST close() when done.
|
||||
"""
|
||||
if not url.startswith(("http://", "https://")):
|
||||
raise ValueError("Only http/https URLs are allowed")
|
||||
width = max(320, min(width, 2560))
|
||||
height = max(240, min(height, 1600))
|
||||
async with self._lock:
|
||||
await self._ensure_playwright()
|
||||
session_id = uuid4().hex
|
||||
profile_key = f"auth-capture-{session_id[:12]}"
|
||||
context = await self._playwright.chromium.launch_persistent_context(
|
||||
str(self._profile_dir(profile_key)),
|
||||
**self._browser_launch_kwargs(width, height),
|
||||
)
|
||||
await self._install_browser_init_scripts(context)
|
||||
# Grant clipboard access for the page origin
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
origin = f"{parsed.scheme}://{parsed.netloc}"
|
||||
await context.grant_permissions(["clipboard-read", "clipboard-write"], origin=origin)
|
||||
except Exception:
|
||||
logger.debug("clipboard permission grant failed (non-fatal)")
|
||||
page = context.pages[0] if context.pages else await context.new_page()
|
||||
tab_id = uuid4().hex
|
||||
tab = BrowserTab(id=tab_id, page=page, created_at=asyncio.get_event_loop().time())
|
||||
session = BrowserSession(
|
||||
id=session_id,
|
||||
custom_page_id=0,
|
||||
profile_key=profile_key,
|
||||
context=context,
|
||||
tabs={tab_id: tab},
|
||||
active_tab_id=tab_id,
|
||||
lock=asyncio.Lock(),
|
||||
captured_headers=[],
|
||||
)
|
||||
self._sessions[session.id] = session
|
||||
self._touch(session.id)
|
||||
# Register listeners for the initial tab
|
||||
self._setup_tab_listeners(session, page)
|
||||
# Register page capture
|
||||
context.on("page", lambda p: self._handle_new_page(session, p))
|
||||
# Start CDP network capture BEFORE the initial page load,
|
||||
# so we capture login redirects and auth headers from the start.
|
||||
await self._start_cdp_capture(session)
|
||||
try:
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=45000)
|
||||
except Exception:
|
||||
await self.close(session.id)
|
||||
raise
|
||||
return session
|
||||
|
||||
async def _start_cdp_capture(self, session: BrowserSession) -> None:
|
||||
"""Enable CDP Network domain and capture Authorization headers."""
|
||||
try:
|
||||
cdp = await session.context.new_cdp_session(session.page)
|
||||
await cdp.send("Network.enable")
|
||||
|
||||
def on_request(params: dict) -> None:
|
||||
headers = params.get("request", {}).get("headers", {})
|
||||
auth = headers.get("authorization") or headers.get("Authorization")
|
||||
api_key = headers.get("x-api-key") or headers.get("X-API-Key")
|
||||
url = params.get("request", {}).get("url", "")
|
||||
if auth:
|
||||
session.captured_headers.append({
|
||||
"type": "authorization",
|
||||
"value": auth,
|
||||
"url": url,
|
||||
})
|
||||
if api_key:
|
||||
session.captured_headers.append({
|
||||
"type": "api_key",
|
||||
"value": api_key,
|
||||
"url": url,
|
||||
})
|
||||
|
||||
cdp.on("Network.requestWillBeSent", on_request)
|
||||
session.cdp_session = cdp
|
||||
except Exception as exc:
|
||||
logger.debug("CDP capture not available: %s", exc)
|
||||
|
||||
async def _save_session_state(self, session: BrowserSession) -> None:
|
||||
if not session.profile_key or session.profile_key.startswith("auth-capture-"):
|
||||
return
|
||||
try:
|
||||
state = await session.context.storage_state()
|
||||
cookies_path = self._cookies_path(session.profile_key)
|
||||
import json
|
||||
import tempfile
|
||||
import os
|
||||
# Ensure parent directories exist
|
||||
cookies_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temp_fd, temp_path = tempfile.mkstemp(dir=str(cookies_path.parent))
|
||||
try:
|
||||
with os.fdopen(temp_fd, 'w', encoding='utf-8') as f:
|
||||
json.dump(state, f, ensure_ascii=False, indent=2)
|
||||
os.replace(temp_path, cookies_path)
|
||||
except Exception:
|
||||
try:
|
||||
os.unlink(temp_path)
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.debug("failed to save session state for %s: %s", session.profile_key, exc)
|
||||
|
||||
async def _restore_session_state(self, context: Any, profile_key: str) -> None:
|
||||
if profile_key.startswith("auth-capture-"):
|
||||
return
|
||||
cookies_path = self._cookies_path(profile_key)
|
||||
if not cookies_path.exists() or cookies_path.stat().st_size == 0:
|
||||
return
|
||||
try:
|
||||
import json
|
||||
import time
|
||||
with open(cookies_path, 'r', encoding='utf-8') as f:
|
||||
state = json.load(f)
|
||||
cookies = state.get("cookies", [])
|
||||
if cookies:
|
||||
now = time.time()
|
||||
valid_cookies = []
|
||||
for c in cookies:
|
||||
expires = c.get("expires")
|
||||
if expires is not None and expires > 0 and expires <= now:
|
||||
continue
|
||||
if expires is not None and expires <= 0:
|
||||
c.pop("expires", None)
|
||||
valid_cookies.append(c)
|
||||
if valid_cookies:
|
||||
await context.add_cookies(valid_cookies)
|
||||
logger.info("restored %d cookies for profile %s", len(valid_cookies), profile_key)
|
||||
|
||||
# 还原 LocalStorage
|
||||
origins = state.get("origins", [])
|
||||
if origins:
|
||||
origins_json = json.dumps(origins)
|
||||
init_script = f"""
|
||||
(() => {{
|
||||
try {{
|
||||
const origins = {origins_json};
|
||||
const currentOrigin = window.location.origin;
|
||||
const target = origins.find(o => o.origin === currentOrigin);
|
||||
if (target && target.localStorage) {{
|
||||
for (const item of target.localStorage) {{
|
||||
try {{
|
||||
window.localStorage.setItem(item.name, item.value);
|
||||
}} catch (e) {{
|
||||
console.error('Failed to restore localStorage key', item.name, e);
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}} catch (err) {{
|
||||
console.error('LocalStorage restore initialization script failed', err);
|
||||
}}
|
||||
}})();
|
||||
"""
|
||||
await context.add_init_script(init_script)
|
||||
logger.info("registered LocalStorage init script for profile %s (origins: %d)", profile_key, len(origins))
|
||||
except Exception as exc:
|
||||
logger.warning("failed to restore cookies/state for profile %s: %s", profile_key, exc)
|
||||
|
||||
|
||||
browser_sessions = BrowserSessionService()
|
||||
@@ -8,4 +8,3 @@ apscheduler==3.10.4
|
||||
python-dotenv==1.0.1
|
||||
pydantic-settings==2.6.1
|
||||
python-multipart==0.0.20
|
||||
playwright==1.52.0
|
||||
|
||||
@@ -1,425 +0,0 @@
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from app.config import get_settings
|
||||
from app.routers.auth_capture import _sanitize_candidate
|
||||
from app.services.browser_session_service import BrowserSessionService
|
||||
|
||||
|
||||
class FakeLocator:
|
||||
def __init__(self, *, visible=True, count=1):
|
||||
self._visible = list(visible) if isinstance(visible, list) else [visible]
|
||||
self._count = count
|
||||
self.filled = []
|
||||
self.clicked = 0
|
||||
self.timeouts = []
|
||||
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
async def count(self):
|
||||
return self._count
|
||||
|
||||
async def is_visible(self, timeout=0):
|
||||
self.timeouts.append(timeout)
|
||||
if not self._visible:
|
||||
return False
|
||||
if len(self._visible) == 1:
|
||||
return self._visible[0]
|
||||
return self._visible.pop(0)
|
||||
|
||||
async def fill(self, value, timeout=0):
|
||||
self.filled.append((value, timeout))
|
||||
|
||||
async def click(self, timeout=0):
|
||||
self.clicked += 1
|
||||
|
||||
|
||||
class FakePage:
|
||||
url = "https://example.test/login"
|
||||
|
||||
def __init__(self, locators):
|
||||
self.locators = locators
|
||||
self.queries = []
|
||||
|
||||
def locator(self, selector):
|
||||
self.queries.append(selector)
|
||||
return self.locators.get(selector, FakeLocator(visible=False, count=0))
|
||||
|
||||
|
||||
def run(coro):
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def test_autofill_retries_until_delayed_fields_are_visible():
|
||||
service = BrowserSessionService()
|
||||
username = FakeLocator(visible=[False, True])
|
||||
password = FakeLocator(visible=True)
|
||||
submit = FakeLocator(visible=True)
|
||||
page = FakePage({
|
||||
"#user": username,
|
||||
"#pass": password,
|
||||
"#submit": submit,
|
||||
})
|
||||
|
||||
run(service._autofill_login(
|
||||
page,
|
||||
{
|
||||
"enabled": True,
|
||||
"username": "alice",
|
||||
"password": "secret",
|
||||
"username_selector": "#user",
|
||||
"password_selector": "#pass",
|
||||
"submit_selector": "#submit",
|
||||
},
|
||||
max_wait_seconds=1,
|
||||
poll_interval_seconds=0,
|
||||
))
|
||||
|
||||
assert page.queries[0] == "#user"
|
||||
assert "#pass" in page.queries
|
||||
assert "input[type='password']" not in page.queries
|
||||
assert username.filled == [("alice", 3000)]
|
||||
assert password.filled == [("secret", 3000)]
|
||||
assert submit.clicked == 1
|
||||
|
||||
|
||||
def test_autofill_returns_without_selectors_when_disabled_or_missing_credentials():
|
||||
service = BrowserSessionService()
|
||||
|
||||
disabled_page = FakePage({"#user": FakeLocator()})
|
||||
run(service._autofill_login(
|
||||
disabled_page,
|
||||
{"enabled": False, "username": "alice", "password": "secret"},
|
||||
max_wait_seconds=1,
|
||||
poll_interval_seconds=0,
|
||||
))
|
||||
assert disabled_page.queries == []
|
||||
|
||||
missing_password_page = FakePage({"#user": FakeLocator()})
|
||||
run(service._autofill_login(
|
||||
missing_password_page,
|
||||
{"enabled": True, "username": "alice", "password": ""},
|
||||
max_wait_seconds=1,
|
||||
poll_interval_seconds=0,
|
||||
))
|
||||
assert missing_password_page.queries == []
|
||||
|
||||
|
||||
def test_sanitize_candidate_strips_secret_fields_but_keeps_metadata():
|
||||
sanitized = _sanitize_candidate({
|
||||
"type": "cookie",
|
||||
"source": "cookie:session",
|
||||
"value": "Bearer secret-token",
|
||||
"preview": "Bearer s…token",
|
||||
"label": "session cookie",
|
||||
"confidence": 90,
|
||||
"cookie_name": "session",
|
||||
"cookie_value": "secret-cookie",
|
||||
"domain": "example.test",
|
||||
})
|
||||
|
||||
assert sanitized == {
|
||||
"type": "cookie",
|
||||
"source": "cookie:session",
|
||||
"preview": "Bearer s…token",
|
||||
"label": "session cookie",
|
||||
"confidence": 90,
|
||||
"cookie_name": "session",
|
||||
"domain": "example.test",
|
||||
}
|
||||
|
||||
|
||||
def test_cookies_path_mapping():
|
||||
import tempfile
|
||||
import shutil
|
||||
service = BrowserSessionService()
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
original_dir = get_settings().browser_profiles_dir
|
||||
get_settings().browser_profiles_dir = temp_dir
|
||||
try:
|
||||
profile_key = "test-profile-123"
|
||||
expected_path = Path(service._cookies_path(profile_key))
|
||||
assert expected_path.name == "session-cookies.json"
|
||||
assert expected_path.parent == Path(service._profile_dir(profile_key))
|
||||
finally:
|
||||
get_settings().browser_profiles_dir = original_dir
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def test_screenshot_throttled_save():
|
||||
import tempfile
|
||||
import shutil
|
||||
service = BrowserSessionService()
|
||||
|
||||
# 准备临时目录并 mock settings.browser_profiles_dir
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
original_dir = get_settings().browser_profiles_dir
|
||||
get_settings().browser_profiles_dir = temp_dir
|
||||
|
||||
try:
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from app.services.browser_session_service import BrowserSession, BrowserTab
|
||||
|
||||
# Mock Context & Page
|
||||
fake_context = AsyncMock()
|
||||
fake_page = MagicMock()
|
||||
fake_page.is_closed = MagicMock(return_value=False)
|
||||
fake_page.screenshot = AsyncMock(return_value=b"screenshot-bytes")
|
||||
|
||||
session = BrowserSession(
|
||||
id="session123",
|
||||
custom_page_id=1,
|
||||
profile_key="page-1-test",
|
||||
context=fake_context,
|
||||
tabs={"main": BrowserTab(id="main", page=fake_page, created_at=0.0)},
|
||||
active_tab_id="main",
|
||||
lock=asyncio.Lock(),
|
||||
last_saved_state_at=0.0
|
||||
)
|
||||
service._sessions[session.id] = session
|
||||
|
||||
# 第一次调用 screenshot: 触发存储
|
||||
res1 = run(service.screenshot(session.id))
|
||||
assert res1 == b"screenshot-bytes"
|
||||
assert fake_context.storage_state.call_count == 1
|
||||
|
||||
# 记录第一次保存后的时间戳
|
||||
first_save_time = session.last_saved_state_at
|
||||
assert first_save_time > 0
|
||||
|
||||
# 第二次立即调用 screenshot: 应该因为限流 10s 被跳过，不增加 call_count
|
||||
res2 = run(service.screenshot(session.id))
|
||||
assert res2 == b"screenshot-bytes"
|
||||
assert fake_context.storage_state.call_count == 1
|
||||
|
||||
# 模拟 11 秒后（防抖时间已过）再度截图
|
||||
session.last_saved_state_at = first_save_time - 11.0
|
||||
res3 = run(service.screenshot(session.id))
|
||||
assert res3 == b"screenshot-bytes"
|
||||
assert fake_context.storage_state.call_count == 2
|
||||
|
||||
# 测试临时 auth-capture 会话不触发任何 state 存储
|
||||
ephemeral_session = BrowserSession(
|
||||
id="session-eph",
|
||||
custom_page_id=0,
|
||||
profile_key="auth-capture-xyz",
|
||||
context=fake_context,
|
||||
tabs={"main": BrowserTab(id="main", page=fake_page, created_at=0.0)},
|
||||
active_tab_id="main",
|
||||
lock=asyncio.Lock(),
|
||||
last_saved_state_at=0.0
|
||||
)
|
||||
service._sessions[ephemeral_session.id] = ephemeral_session
|
||||
run(service.screenshot(ephemeral_session.id))
|
||||
# 它的 call_count 依然是 2，没有增加
|
||||
assert fake_context.storage_state.call_count == 2
|
||||
|
||||
finally:
|
||||
get_settings().browser_profiles_dir = original_dir
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def test_close_saves_state_and_cleans_up():
|
||||
import tempfile
|
||||
import shutil
|
||||
service = BrowserSessionService()
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
original_dir = get_settings().browser_profiles_dir
|
||||
get_settings().browser_profiles_dir = temp_dir
|
||||
|
||||
try:
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from app.services.browser_session_service import BrowserSession, BrowserTab
|
||||
|
||||
fake_context = AsyncMock()
|
||||
fake_page = MagicMock()
|
||||
fake_page.is_closed = MagicMock(return_value=False)
|
||||
|
||||
import time
|
||||
session = BrowserSession(
|
||||
id="session_close",
|
||||
custom_page_id=2,
|
||||
profile_key="page-2-test",
|
||||
context=fake_context,
|
||||
tabs={"main": BrowserTab(id="main", page=fake_page, created_at=0.0)},
|
||||
active_tab_id="main",
|
||||
lock=asyncio.Lock(),
|
||||
last_saved_state_at=time.monotonic() # 此时在限流内
|
||||
)
|
||||
service._sessions[session.id] = session
|
||||
|
||||
# 即使在限流时间内，close 也必须强制保存
|
||||
run(service.close(session.id))
|
||||
assert fake_context.storage_state.call_count == 1
|
||||
assert fake_context.close.call_count == 1
|
||||
|
||||
# 测试 ephemeral 会话在 close 时不应该保存 state，并且其 profile_dir 应当被清理，导致 cookies json 不复存在
|
||||
eph_context = AsyncMock()
|
||||
eph_page = MagicMock()
|
||||
eph_page.is_closed = MagicMock(return_value=False)
|
||||
eph_session = BrowserSession(
|
||||
id="session_eph_close",
|
||||
custom_page_id=0,
|
||||
profile_key="auth-capture-abc",
|
||||
context=eph_context,
|
||||
tabs={"main": BrowserTab(id="main", page=eph_page, created_at=0.0)},
|
||||
active_tab_id="main",
|
||||
lock=asyncio.Lock(),
|
||||
last_saved_state_at=0.0
|
||||
)
|
||||
service._sessions[eph_session.id] = eph_session
|
||||
|
||||
# 先手动创建一个假 session-cookies.json
|
||||
eph_cookies_path = service._cookies_path(eph_session.profile_key)
|
||||
eph_cookies_path.write_text("{}")
|
||||
assert eph_cookies_path.exists()
|
||||
|
||||
run(service.close(eph_session.id))
|
||||
# ephemeral close 时不保存，所以 call_count 依然是 0
|
||||
assert eph_context.storage_state.call_count == 0
|
||||
assert eph_context.close.call_count == 1
|
||||
# 但对应的 profile 目录已被删除，文件自然不复存在
|
||||
assert not eph_cookies_path.exists()
|
||||
|
||||
finally:
|
||||
get_settings().browser_profiles_dir = original_dir
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def test_restore_session_state_decoding_and_inject():
|
||||
import json
|
||||
import tempfile
|
||||
import shutil
|
||||
import time
|
||||
service = BrowserSessionService()
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
original_dir = get_settings().browser_profiles_dir
|
||||
get_settings().browser_profiles_dir = temp_dir
|
||||
|
||||
try:
|
||||
from unittest.mock import AsyncMock
|
||||
fake_context = AsyncMock()
|
||||
profile_key = "test-restore-profile"
|
||||
|
||||
# 准备假 cookies.json，包含 cookies 和 origins/localStorage
|
||||
cookies_path = service._cookies_path(profile_key)
|
||||
|
||||
now = time.time()
|
||||
fake_state = {
|
||||
"cookies": [
|
||||
{
|
||||
"name": "valid_persistent",
|
||||
"value": "123",
|
||||
"domain": "example.test",
|
||||
"path": "/",
|
||||
"expires": now + 3600 # 未过期
|
||||
},
|
||||
{
|
||||
"name": "expired_cookie",
|
||||
"value": "456",
|
||||
"domain": "example.test",
|
||||
"path": "/",
|
||||
"expires": now - 3600 # 已过期
|
||||
},
|
||||
{
|
||||
"name": "session_cookie",
|
||||
"value": "789",
|
||||
"domain": "example.test",
|
||||
"path": "/",
|
||||
"expires": -1 # session cookie，应被保留并剔除 expires 字段
|
||||
}
|
||||
],
|
||||
"origins": [
|
||||
{
|
||||
"origin": "https://example.test",
|
||||
"localStorage": [
|
||||
{
|
||||
"name": "theme",
|
||||
"value": "dark"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
with open(cookies_path, "w", encoding='utf-8') as f:
|
||||
json.dump(fake_state, f)
|
||||
|
||||
# 运行还原方法
|
||||
run(service._restore_session_state(fake_context, profile_key))
|
||||
|
||||
# 检查是否成功调用 add_cookies
|
||||
assert fake_context.add_cookies.call_count == 1
|
||||
|
||||
# 检查过滤后的 cookies 内容
|
||||
injected_cookies = fake_context.add_cookies.call_args[0][0]
|
||||
assert len(injected_cookies) == 2
|
||||
|
||||
names = [c["name"] for c in injected_cookies]
|
||||
assert "valid_persistent" in names
|
||||
assert "session_cookie" in names
|
||||
assert "expired_cookie" not in names
|
||||
|
||||
# 校验 session_cookie 是否成功移除了 expires
|
||||
session_c = next(c for c in injected_cookies if c["name"] == "session_cookie")
|
||||
assert "expires" not in session_c
|
||||
|
||||
# 检查是否成功调用了 add_init_script (用于还原 LocalStorage)
|
||||
assert fake_context.add_init_script.call_count == 1
|
||||
init_script = fake_context.add_init_script.call_args[0][0]
|
||||
assert "window.localStorage.setItem" in init_script
|
||||
assert "theme" in init_script
|
||||
assert "dark" in init_script
|
||||
|
||||
finally:
|
||||
get_settings().browser_profiles_dir = original_dir
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def test_websocket_event_saves_state():
|
||||
import tempfile
|
||||
import shutil
|
||||
service = BrowserSessionService()
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
original_dir = get_settings().browser_profiles_dir
|
||||
get_settings().browser_profiles_dir = temp_dir
|
||||
|
||||
try:
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from app.services.browser_session_service import BrowserSession, BrowserTab
|
||||
|
||||
fake_context = AsyncMock()
|
||||
fake_page = MagicMock()
|
||||
fake_page.is_closed = MagicMock(return_value=False)
|
||||
fake_page.mouse = MagicMock()
|
||||
fake_page.mouse.click = AsyncMock()
|
||||
|
||||
session = BrowserSession(
|
||||
id="session_ws",
|
||||
custom_page_id=3,
|
||||
profile_key="page-3-ws-test",
|
||||
context=fake_context,
|
||||
tabs={"main": BrowserTab(id="main", page=fake_page, created_at=0.0)},
|
||||
active_tab_id="main",
|
||||
lock=asyncio.Lock(),
|
||||
last_saved_state_at=0.0
|
||||
)
|
||||
service._sessions[session.id] = session
|
||||
|
||||
# 即使 include_state=False，也应当在 5 秒节流到期后保存状态
|
||||
run(service.event(
|
||||
session_id=session.id,
|
||||
event_type="click",
|
||||
payload={"x": 10.0, "y": 20.0},
|
||||
include_state=False
|
||||
))
|
||||
|
||||
# storage_state 应该被调用，说明保存成功触发了
|
||||
assert fake_context.storage_state.call_count == 1
|
||||
assert session.last_saved_state_at > 0
|
||||
|
||||
finally:
|
||||
get_settings().browser_profiles_dir = original_dir
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
|
||||
|
||||
@@ -1,192 +0,0 @@
|
||||
import asyncio
|
||||
import json
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from app.services.browser_session_service import BrowserSessionService, BrowserSession, BrowserTab, BrowserSessionError
|
||||
|
||||
@pytest.fixture
|
||||
def service():
|
||||
return BrowserSessionService()
|
||||
|
||||
@pytest.fixture
|
||||
def session(service):
|
||||
fake_context = AsyncMock()
|
||||
fake_page = AsyncMock()
|
||||
# Playwright's page.on is synchronous
|
||||
fake_page.on = MagicMock()
|
||||
fake_page.is_closed = MagicMock(return_value=False)
|
||||
fake_page.url = "https://initial.test"
|
||||
fake_page.title = AsyncMock(return_value="Initial Tab")
|
||||
|
||||
tab_id = "tab1"
|
||||
tab = BrowserTab(id=tab_id, page=fake_page, created_at=100.0)
|
||||
|
||||
sess = BrowserSession(
|
||||
id="sess123",
|
||||
custom_page_id=1,
|
||||
profile_key="profile1",
|
||||
context=fake_context,
|
||||
tabs={tab_id: tab},
|
||||
active_tab_id=tab_id,
|
||||
lock=asyncio.Lock(),
|
||||
)
|
||||
service._sessions[sess.id] = sess
|
||||
return sess
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tab_capture(service, session):
|
||||
new_page = AsyncMock()
|
||||
new_page.on = MagicMock()
|
||||
new_page.is_closed = MagicMock(return_value=False)
|
||||
new_page.url = "https://new.test"
|
||||
new_page.title = AsyncMock(return_value="New Tab")
|
||||
new_page.bring_to_front = AsyncMock()
|
||||
|
||||
# Simulate page capture
|
||||
service._handle_new_page(session, new_page)
|
||||
|
||||
# Wait for the background registration task to finish
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
assert len(session.tabs) == 2
|
||||
assert session.active_tab_id != "tab1"
|
||||
new_tab_id = session.active_tab_id
|
||||
assert session.tabs[new_tab_id].page == new_page
|
||||
assert session.tab_revision == 1
|
||||
|
||||
# Wait a bit for the background task _init_new_tab to finish if possible,
|
||||
# though it's mocked anyway.
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_activate_tab(service, session):
|
||||
new_page = AsyncMock()
|
||||
new_page.is_closed = MagicMock(return_value=False)
|
||||
new_page.bring_to_front = AsyncMock()
|
||||
tab2_id = "tab2"
|
||||
session.tabs[tab2_id] = BrowserTab(id=tab2_id, page=new_page, created_at=200.0)
|
||||
|
||||
await service.activate_tab(session.id, tab2_id)
|
||||
|
||||
assert session.active_tab_id == tab2_id
|
||||
assert new_page.bring_to_front.call_count == 1
|
||||
assert session.tab_revision == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_tab_safety(service, session):
|
||||
# Cannot close last tab
|
||||
with pytest.raises(ValueError, match="cannot close the last tab"):
|
||||
await service.close_tab(session.id, "tab1")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_active_tab_fallback(service, session):
|
||||
# Setup tab2
|
||||
page2 = AsyncMock()
|
||||
page2.is_closed = MagicMock(return_value=False)
|
||||
page2.bring_to_front = AsyncMock()
|
||||
tab2_id = "tab2"
|
||||
session.tabs[tab2_id] = BrowserTab(id=tab2_id, page=page2, created_at=200.0)
|
||||
|
||||
# Active is tab2
|
||||
session.active_tab_id = tab2_id
|
||||
|
||||
# Close active tab2
|
||||
await service.close_tab(session.id, tab2_id)
|
||||
|
||||
assert len(session.tabs) == 1
|
||||
assert session.active_tab_id == "tab1"
|
||||
assert tab2_id not in session.tabs
|
||||
assert session.tabs["tab1"].page.bring_to_front.call_count == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_state_includes_tabs(service, session):
|
||||
# Setup tab2
|
||||
page2 = AsyncMock()
|
||||
page2.is_closed = MagicMock(return_value=False)
|
||||
page2.url = "https://tab2.test"
|
||||
page2.title = AsyncMock(return_value="Tab 2")
|
||||
tab2_id = "tab2"
|
||||
session.tabs[tab2_id] = BrowserTab(id=tab2_id, page=page2, created_at=200.0)
|
||||
|
||||
state = await service.state(session.id)
|
||||
|
||||
assert state["id"] == session.id
|
||||
assert state["active_tab_id"] == "tab1"
|
||||
assert len(state["tabs"]) == 2
|
||||
|
||||
# Tabs should be sorted by created_at
|
||||
assert state["tabs"][0]["id"] == "tab1"
|
||||
assert state["tabs"][1]["id"] == "tab2"
|
||||
assert state["tabs"][0]["title"] == "Initial Tab"
|
||||
assert state["tabs"][1]["url"] == "https://tab2.test"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_open_prunes_closed_tab(service, session):
|
||||
# Setup tab2 and make it active
|
||||
page2 = AsyncMock()
|
||||
page2.is_closed = MagicMock(return_value=True) # Page 2 is closed
|
||||
tab2_id = "tab2"
|
||||
session.tabs[tab2_id] = BrowserTab(id=tab2_id, page=page2, created_at=200.0)
|
||||
session.active_tab_id = tab2_id
|
||||
|
||||
# Calling any interaction method should trigger _ensure_open
|
||||
# and fallback to tab1
|
||||
await service.screenshot(session.id)
|
||||
|
||||
assert session.active_tab_id == "tab1"
|
||||
assert tab2_id not in session.tabs
|
||||
assert session.tab_revision == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_open_discards_session_if_all_tabs_closed(service, session):
|
||||
session.tabs["tab1"].page.is_closed = MagicMock(return_value=True)
|
||||
|
||||
with pytest.raises(BrowserSessionError, match="all browser pages are closed"):
|
||||
await service.screenshot(session.id)
|
||||
|
||||
assert session.id not in service._sessions
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tab_revision_bumps_on_events(service, session):
|
||||
# Setup listeners for the initial tab
|
||||
service._setup_tab_listeners(session, session.tabs["tab1"].page)
|
||||
|
||||
# Extract the "load" listener callback
|
||||
calls = session.tabs["tab1"].page.on.call_args_list
|
||||
load_callback = next(c[0][1] for c in calls if c[0][0] == "load")
|
||||
|
||||
initial_revision = session.tab_revision
|
||||
load_callback()
|
||||
assert session.tab_revision == initial_revision + 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_state_concurrency_with_popup(service, session):
|
||||
# Setup: page.title() will trigger a new page registration in background
|
||||
triggered = False
|
||||
async def mock_title():
|
||||
nonlocal triggered
|
||||
# Only trigger popup once (state() calls title twice: once for tabs list, once for top-level)
|
||||
if not triggered:
|
||||
triggered = True
|
||||
# Simulate a popup arriving while title is being fetched
|
||||
new_page = AsyncMock()
|
||||
new_page.on = MagicMock()
|
||||
new_page.is_closed = MagicMock(return_value=False)
|
||||
new_page.url = "https://new.test"
|
||||
new_page.bring_to_front = AsyncMock()
|
||||
service._handle_new_page(session, new_page)
|
||||
# Yield to let the registration task start (it will block on the lock)
|
||||
await asyncio.sleep(0.01)
|
||||
return "Initial Tab"
|
||||
|
||||
session.tabs["tab1"].page.title = mock_title
|
||||
|
||||
# Call state() which takes the lock and calls _session_state (which calls mock_title)
|
||||
state = await service.state(session.id)
|
||||
|
||||
assert len(state["tabs"]) == 1 # Still 1 because popup registration is waiting for lock
|
||||
|
||||
# Yield to let registration task finish after state() released the lock
|
||||
await asyncio.sleep(0.1)
|
||||
assert len(session.tabs) == 2
|
||||
assert session.tab_revision > 0
|
||||
@@ -36,7 +36,7 @@ def test_create_page_auto_enables_autofill_when_credentials_are_saved(db_session
|
||||
custom_pages.CustomPageCreate(
|
||||
name="Login page",
|
||||
url="https://example.test/login",
|
||||
access_mode="remote_browser",
|
||||
access_mode="direct",
|
||||
login_username="alice",
|
||||
login_password="secret",
|
||||
),
|
||||
@@ -53,7 +53,7 @@ def test_create_page_respects_explicit_autofill_disable(db_session):
|
||||
custom_pages.CustomPageCreate(
|
||||
name="Login page",
|
||||
url="https://example.test/login",
|
||||
access_mode="remote_browser",
|
||||
access_mode="direct",
|
||||
login_username="alice",
|
||||
login_password="secret",
|
||||
login_autofill_enabled=False,
|
||||
@@ -69,7 +69,7 @@ def test_update_page_auto_enables_autofill_when_new_password_is_saved(db_session
|
||||
page = CustomPage(
|
||||
name="Login page",
|
||||
url="https://example.test/login",
|
||||
access_mode="remote_browser",
|
||||
access_mode="direct",
|
||||
login_username="alice",
|
||||
login_password="old-secret",
|
||||
login_autofill_enabled=False,
|
||||
@@ -95,7 +95,7 @@ def test_update_page_keeps_autofill_disabled_when_existing_password_is_kept(db_s
|
||||
page = CustomPage(
|
||||
name="Login page",
|
||||
url="https://example.test/login",
|
||||
access_mode="remote_browser",
|
||||
access_mode="direct",
|
||||
login_username="alice",
|
||||
login_password="secret",
|
||||
login_autofill_enabled=False,
|
||||
@@ -118,7 +118,7 @@ def test_update_page_respects_explicit_autofill_disable(db_session):
|
||||
page = CustomPage(
|
||||
name="Login page",
|
||||
url="https://example.test/login",
|
||||
access_mode="remote_browser",
|
||||
access_mode="direct",
|
||||
login_username="alice",
|
||||
login_password="secret",
|
||||
login_autofill_enabled=False,
|
||||
@@ -154,7 +154,7 @@ def test_custom_page_migration_backfills_autofill_once(monkeypatch):
|
||||
conn.execute(text(
|
||||
"INSERT INTO custom_pages "
|
||||
"(name, url, icon, sort_order, enabled, use_proxy, access_mode, login_username, login_password, login_autofill_enabled, created_at, updated_at) "
|
||||
"VALUES ('Login page', 'https://example.test/login', 'Link', 0, 1, 0, 'remote_browser', 'alice', 'secret', 0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)"
|
||||
"VALUES ('Login page', 'https://example.test/login', 'Link', 0, 1, 0, 'direct', 'alice', 'secret', 0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)"
|
||||
))
|
||||
|
||||
database_module._migrate_custom_pages()
|
||||
|
||||
@@ -1,177 +0,0 @@
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from app.database import Base
|
||||
from app.models.custom_page import CustomPage
|
||||
from app.models.upstream import Upstream
|
||||
from app.routers import custom_pages
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def db_session():
|
||||
engine = create_engine(
|
||||
"sqlite://",
|
||||
connect_args={"check_same_thread": False},
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
Base.metadata.create_all(bind=engine)
|
||||
TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
db = TestingSessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
Base.metadata.drop_all(bind=engine)
|
||||
|
||||
|
||||
def _linked_page(db, upstream: Upstream) -> CustomPage:
|
||||
db.add(upstream)
|
||||
db.commit()
|
||||
db.refresh(upstream)
|
||||
page = CustomPage(
|
||||
name="Login",
|
||||
url="https://meow.example/login",
|
||||
access_mode="remote_browser",
|
||||
linked_upstream_id=upstream.id,
|
||||
)
|
||||
db.add(page)
|
||||
db.commit()
|
||||
db.refresh(page)
|
||||
return page
|
||||
|
||||
|
||||
def _install_refresh_fakes(monkeypatch, candidates: list[dict], calls: list[dict]):
|
||||
monkeypatch.setattr(custom_pages.browser_sessions, "find_by_page_id", lambda _pid: object())
|
||||
|
||||
async def fake_extract_all(_session):
|
||||
return {"candidates": candidates}
|
||||
|
||||
monkeypatch.setattr(custom_pages, "extract_all", fake_extract_all)
|
||||
|
||||
class FakeUpstreamClient:
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def get_available_groups(self, endpoint):
|
||||
calls.append({"endpoint": endpoint, **self.kwargs})
|
||||
return [{"id": "default", "name": "Default"}]
|
||||
|
||||
import app.services.upstream_client as upstream_client_module
|
||||
|
||||
monkeypatch.setattr(upstream_client_module, "UpstreamClient", FakeUpstreamClient)
|
||||
|
||||
|
||||
def test_refresh_auth_sub2api_prefers_bearer_over_cookie_bundle(db_session, monkeypatch):
|
||||
upstream = Upstream(
|
||||
name="Meow",
|
||||
base_url="https://api.saki.lat",
|
||||
api_prefix="/api/v1",
|
||||
auth_type="login_password",
|
||||
auth_config_json=json.dumps({
|
||||
"email": "alice@example.test",
|
||||
"password": "secret",
|
||||
"login_path": "/auth/login",
|
||||
}),
|
||||
groups_endpoint="/groups/available",
|
||||
rate_endpoint="/groups/rates",
|
||||
)
|
||||
page = _linked_page(db_session, upstream)
|
||||
calls: list[dict] = []
|
||||
_install_refresh_fakes(monkeypatch, [
|
||||
{"type": "cookie_bundle", "value": "cf_clearance=cf; session=s", "cookie_count": 2},
|
||||
{"type": "bearer_token", "value": "jwt.header.payload", "source": "localStorage.auth_token"},
|
||||
], calls)
|
||||
|
||||
response = asyncio.run(custom_pages.refresh_auth(page.id, db_session, object()))
|
||||
|
||||
db_session.refresh(upstream)
|
||||
cfg = json.loads(upstream.auth_config_json)
|
||||
assert response.success is True
|
||||
assert upstream.auth_type == "bearer"
|
||||
assert cfg["token"] == "jwt.header.payload"
|
||||
assert "cookie_string" not in cfg
|
||||
assert upstream.api_prefix == "/api/v1"
|
||||
assert upstream.groups_endpoint == "/groups/available"
|
||||
assert calls[0]["auth_type"] == "bearer"
|
||||
assert calls[0]["endpoint"] == "/groups/available"
|
||||
|
||||
|
||||
def test_refresh_auth_sub2api_rejects_cookie_only_capture(db_session, monkeypatch):
|
||||
upstream = Upstream(
|
||||
name="Meow",
|
||||
base_url="https://api.saki.lat",
|
||||
api_prefix="/api/v1",
|
||||
auth_type="login_password",
|
||||
auth_config_json=json.dumps({"login_path": "/auth/login"}),
|
||||
groups_endpoint="/groups/available",
|
||||
rate_endpoint="/groups/rates",
|
||||
)
|
||||
page = _linked_page(db_session, upstream)
|
||||
_install_refresh_fakes(monkeypatch, [
|
||||
{"type": "cookie_bundle", "value": "cf_clearance=cf; session=s", "cookie_count": 2},
|
||||
], [])
|
||||
|
||||
response = asyncio.run(custom_pages.refresh_auth(page.id, db_session, object()))
|
||||
|
||||
db_session.refresh(upstream)
|
||||
cfg = json.loads(upstream.auth_config_json)
|
||||
assert response.success is False
|
||||
assert "Sub2API 需要 Bearer Token" in response.message
|
||||
assert upstream.auth_type == "login_password"
|
||||
assert "cookie_string" not in cfg
|
||||
|
||||
|
||||
def test_refresh_auth_new_api_user_uses_cookie_bundle_and_resets_user_endpoints(db_session, monkeypatch):
|
||||
upstream = Upstream(
|
||||
name="New API User",
|
||||
base_url="https://newapi.example",
|
||||
api_prefix="/api/v1",
|
||||
auth_type="login_password",
|
||||
auth_config_json=json.dumps({
|
||||
"email": "alice",
|
||||
"password": "secret",
|
||||
"login_path": "/api/user/login",
|
||||
}),
|
||||
groups_endpoint="/groups/available",
|
||||
rate_endpoint="/groups/rates",
|
||||
)
|
||||
page = _linked_page(db_session, upstream)
|
||||
calls: list[dict] = []
|
||||
_install_refresh_fakes(monkeypatch, [
|
||||
{
|
||||
"type": "cookie_bundle",
|
||||
"value": "cf_clearance=cf; session=s",
|
||||
"cookie_count": 2,
|
||||
"new_api_user": "42",
|
||||
},
|
||||
{"type": "bearer_token", "value": "jwt.header.payload"},
|
||||
], calls)
|
||||
|
||||
response = asyncio.run(custom_pages.refresh_auth(page.id, db_session, object()))
|
||||
|
||||
db_session.refresh(upstream)
|
||||
cfg = json.loads(upstream.auth_config_json)
|
||||
assert response.success is True
|
||||
assert upstream.auth_type == "cookie"
|
||||
assert cfg["cookie_string"] == "cf_clearance=cf; session=s"
|
||||
assert cfg["new_api_user"] == "42"
|
||||
assert upstream.api_prefix == ""
|
||||
assert upstream.groups_endpoint == "/api/user/self/groups"
|
||||
assert upstream.rate_endpoint == "/api/user/self/groups"
|
||||
assert calls[0]["auth_type"] == "cookie"
|
||||
assert calls[0]["endpoint"] == "/api/user/self/groups"
|
||||
Reference in New Issue
Block a user