feat: persist finance daily summaries
This commit is contained in:
SmartUp Developer
@@ -41,7 +41,7 @@ def get_db():
|
||||
def init_db():
|
||||
"""Create all tables."""
|
||||
# import models so SQLAlchemy registers them
|
||||
from app.models import admin_user, upstream, snapshot, webhook_config, notification_log, custom_page, website, revoked_token, upstream_key, external_api_log # noqa: F401
|
||||
from app.models import admin_user, upstream, snapshot, webhook_config, notification_log, custom_page, website, revoked_token, upstream_key, external_api_log, finance_daily_summary # noqa: F401
|
||||
Base.metadata.create_all(bind=engine)
|
||||
_ensure_indexes()
|
||||
_migrate_custom_pages()
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Finance daily summary snapshot model.
|
||||
|
||||
Stores pre-computed daily reconciliation results in a single JSON column.
|
||||
This keeps the schema stable: adding new summary fields requires no migration.
|
||||
"""
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Boolean, Date, DateTime, Integer, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class FinanceDailySummary(Base):
|
||||
__tablename__ = "finance_daily_summaries"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
|
||||
stat_date: Mapped[date] = mapped_column(Date, unique=True, nullable=False, index=True)
|
||||
summary_json: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
success: Mapped[bool] = mapped_column(Boolean, default=False, index=True)
|
||||
computed_at: Mapped[datetime] = mapped_column(DateTime, default=_utcnow)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("stat_date", name="uq_finance_summary_date"),
|
||||
)
|
||||
+200
-19
@@ -1,42 +1,223 @@
|
||||
"""Finance router — daily revenue vs cost reconciliation."""
|
||||
"""Finance router — daily revenue vs cost reconciliation with snapshot storage."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
from typing import Any, Optional
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models.upstream import Upstream
|
||||
from app.models.website import Website
|
||||
from app.services.finance_service import get_daily_summary, yesterday_shanghai
|
||||
from app.services.finance_service import (
|
||||
compute_daily_summary,
|
||||
get_or_create_daily_summary,
|
||||
overwrite_daily_summary,
|
||||
today_shanghai,
|
||||
yesterday_shanghai,
|
||||
)
|
||||
from app.utils.auth import get_current_user
|
||||
|
||||
router = APIRouter(prefix="/api/finance", tags=["finance"])
|
||||
|
||||
|
||||
def _resolve_date(date_str: str | None) -> _dt.date:
|
||||
"""Parse optional date string; return yesterday Asia/Shanghai if omitted."""
|
||||
if date_str:
|
||||
try:
|
||||
return _dt.date.fromisoformat(date_str)
|
||||
except ValueError:
|
||||
raise HTTPException(400, f"Invalid date format: {date_str!r}. Expected YYYY-MM-DD.")
|
||||
return yesterday_shanghai()
|
||||
|
||||
|
||||
def _validate_not_future(target: _dt.date) -> None:
|
||||
"""Reject future dates — we can't save snapshots for days that haven't happened."""
|
||||
if target > today_shanghai():
|
||||
raise HTTPException(400, f"Cannot query future date: {target.isoformat()}")
|
||||
|
||||
|
||||
@router.get("/daily-summary")
|
||||
def daily_summary(
|
||||
date: Optional[str] = Query(None, description="YYYY-MM-DD, defaults to yesterday Asia/Shanghai"),
|
||||
date: str | None = Query(None, description="YYYY-MM-DD, defaults to yesterday Asia/Shanghai"),
|
||||
db: Session = Depends(get_db),
|
||||
_: Any = Depends(get_current_user),
|
||||
) -> dict:
|
||||
"""Return daily revenue vs cost summary.
|
||||
"""Return stored daily summary snapshot.
|
||||
|
||||
Never raises for individual item failures — each failed website/upstream
|
||||
is included in the response with status='failed' and an error message.
|
||||
Only raises HTTP errors for bad request parameters.
|
||||
If no snapshot exists for the requested date, computes one on the fly,
|
||||
saves it, and returns it. Never raises for individual item failures.
|
||||
"""
|
||||
if date:
|
||||
try:
|
||||
target = _dt.date.fromisoformat(date)
|
||||
except ValueError:
|
||||
raise HTTPException(400, f"Invalid date format: {date!r}. Expected YYYY-MM-DD.")
|
||||
target = _resolve_date(date)
|
||||
_validate_not_future(target)
|
||||
return get_or_create_daily_summary(db, target)
|
||||
|
||||
|
||||
@router.post("/daily-summary/compare")
|
||||
def compare_daily_summary(
|
||||
date: str | None = Query(None, description="YYYY-MM-DD, defaults to yesterday Asia/Shanghai"),
|
||||
db: Session = Depends(get_db),
|
||||
_: Any = Depends(get_current_user),
|
||||
) -> dict:
|
||||
"""Compare stored snapshot against a fresh real-time computation.
|
||||
|
||||
Returns {stored, current, has_difference, diff_items}. Does NOT write to DB.
|
||||
"""
|
||||
target = _resolve_date(date)
|
||||
_validate_not_future(target)
|
||||
|
||||
from app.models.finance_daily_summary import FinanceDailySummary
|
||||
|
||||
stored_row = db.query(FinanceDailySummary).filter(
|
||||
FinanceDailySummary.stat_date == target
|
||||
).first()
|
||||
|
||||
current = compute_daily_summary(db, target)
|
||||
|
||||
if stored_row:
|
||||
stored_data = json.loads(stored_row.summary_json)
|
||||
stored_data["computed_at"] = stored_row.computed_at.isoformat()
|
||||
stored_data["from_snapshot"] = True
|
||||
diff_items, has_diff = _compute_diff_items(stored_data, current)
|
||||
else:
|
||||
target = yesterday_shanghai()
|
||||
stored_data = None
|
||||
diff_items = []
|
||||
has_diff = True # no stored data = definitely different
|
||||
|
||||
websites = db.query(Website).filter(Website.enabled == True).all()
|
||||
upstreams = db.query(Upstream).filter(Upstream.enabled == True).all()
|
||||
return {
|
||||
"stored": stored_data,
|
||||
"current": current,
|
||||
"has_difference": has_diff,
|
||||
"diff_items": diff_items,
|
||||
}
|
||||
|
||||
return get_daily_summary(websites, upstreams, target)
|
||||
|
||||
@router.post("/daily-summary/overwrite")
|
||||
def overwrite_daily_summary_endpoint(
|
||||
date: str | None = Query(None, description="YYYY-MM-DD, defaults to yesterday Asia/Shanghai"),
|
||||
db: Session = Depends(get_db),
|
||||
_: Any = Depends(get_current_user),
|
||||
) -> dict:
|
||||
"""Recompute and overwrite the stored snapshot for the requested date."""
|
||||
target = _resolve_date(date)
|
||||
_validate_not_future(target)
|
||||
return overwrite_daily_summary(db, target)
|
||||
|
||||
|
||||
def _has_difference(stored: dict, current: dict) -> bool:
|
||||
"""Compare stored vs current summary at the total and item level.
|
||||
|
||||
Items are compared by (id, upstream_type) identity rather than by array
|
||||
position, so that a replaced website/upstream doesn't falsely appear
|
||||
identical, and reordered items don't falsely appear different.
|
||||
"""
|
||||
for key in ("total_revenue", "total_cost", "net_income", "success"):
|
||||
if stored.get(key) != current.get(key):
|
||||
return True
|
||||
for side in ("website_items", "upstream_items"):
|
||||
s_list = stored.get(side, [])
|
||||
c_list = current.get(side, [])
|
||||
if len(s_list) != len(c_list):
|
||||
return True
|
||||
# Build lookup map by (id, upstream_type)
|
||||
s_map = {(_item_id(s), s.get("upstream_type")): s for s in s_list}
|
||||
c_map = {(_item_id(c), c.get("upstream_type")): c for c in c_list}
|
||||
# Check that keys match exactly
|
||||
if s_map.keys() != c_map.keys():
|
||||
return True
|
||||
# Compare each item by its stable identity
|
||||
for key_ident, s_item in s_map.items():
|
||||
c_item = c_map[key_ident]
|
||||
for attr in ("status", "amount", "error"):
|
||||
if s_item.get(attr) != c_item.get(attr):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _item_id(item: dict) -> int | str:
|
||||
"""Return item id, defaulting to a sentinel for items that lack one."""
|
||||
v = item.get("id")
|
||||
return v if v is not None else -1
|
||||
|
||||
|
||||
def _compute_diff_items(stored: dict, current: dict) -> tuple[list[dict], bool]:
|
||||
"""Compare stored vs current and return (diff_items, has_difference).
|
||||
|
||||
diff_items is a list of changes at the item level, each with:
|
||||
side, name, id, upstream_type, old_{status,amount,error}, new_{status,amount,error}.
|
||||
|
||||
has_difference also covers total-level changes and count mismatches.
|
||||
"""
|
||||
diff_items: list[dict] = []
|
||||
has_diff = False
|
||||
|
||||
# Total-level checks
|
||||
for key in ("total_revenue", "total_cost", "net_income", "success"):
|
||||
if stored.get(key) != current.get(key):
|
||||
has_diff = True
|
||||
|
||||
# Item-level checks
|
||||
for side in ("website_items", "upstream_items"):
|
||||
s_list = stored.get(side, [])
|
||||
c_list = current.get(side, [])
|
||||
if len(s_list) != len(c_list):
|
||||
has_diff = True
|
||||
s_map = {(_item_id(s), s.get("upstream_type")): s for s in s_list}
|
||||
c_map = {(_item_id(c), c.get("upstream_type")): c for c in c_list}
|
||||
if s_map.keys() != c_map.keys():
|
||||
has_diff = True
|
||||
# Items in both sides — compare attributes
|
||||
for key_ident, s_item in s_map.items():
|
||||
c_item = c_map.get(key_ident)
|
||||
if c_item is None:
|
||||
continue # already counted as key mismatch
|
||||
for attr in ("status", "amount", "error"):
|
||||
if s_item.get(attr) != c_item.get(attr):
|
||||
has_diff = True
|
||||
diff_items.append({
|
||||
"side": side,
|
||||
"id": _item_id(s_item),
|
||||
"name": s_item.get("name", ""),
|
||||
"upstream_type": s_item.get("upstream_type", ""),
|
||||
"old_status": s_item.get("status"),
|
||||
"new_status": c_item.get("status"),
|
||||
"old_amount": s_item.get("amount"),
|
||||
"new_amount": c_item.get("amount"),
|
||||
"old_error": s_item.get("error"),
|
||||
"new_error": c_item.get("error"),
|
||||
})
|
||||
break # one diff entry per item
|
||||
# Items only in stored (removed)
|
||||
for key_ident in s_map.keys() - c_map.keys():
|
||||
has_diff = True
|
||||
s_item = s_map[key_ident]
|
||||
diff_items.append({
|
||||
"side": side,
|
||||
"id": _item_id(s_item),
|
||||
"name": s_item.get("name", ""),
|
||||
"upstream_type": s_item.get("upstream_type", ""),
|
||||
"old_status": s_item.get("status"),
|
||||
"new_status": None,
|
||||
"old_amount": s_item.get("amount"),
|
||||
"new_amount": None,
|
||||
"old_error": s_item.get("error"),
|
||||
"new_error": None,
|
||||
})
|
||||
# Items only in current (added)
|
||||
for key_ident in c_map.keys() - s_map.keys():
|
||||
has_diff = True
|
||||
c_item = c_map[key_ident]
|
||||
diff_items.append({
|
||||
"side": side,
|
||||
"id": _item_id(c_item),
|
||||
"name": c_item.get("name", ""),
|
||||
"upstream_type": c_item.get("upstream_type", ""),
|
||||
"old_status": None,
|
||||
"new_status": c_item.get("status"),
|
||||
"old_amount": None,
|
||||
"new_amount": c_item.get("amount"),
|
||||
"old_error": None,
|
||||
"new_error": c_item.get("error"),
|
||||
})
|
||||
|
||||
return diff_items, has_diff
|
||||
|
||||
@@ -7,18 +7,22 @@ Architecture:
|
||||
within the client context — retains cookies, 401 refresh, external API log.
|
||||
- Upstream cost (New-API/Nox-API): same UpstreamClient pattern.
|
||||
|
||||
No DB writes. Each item fails independently; failures do NOT contribute to
|
||||
totals, and the UI must treat any failed item as incomplete accounting data.
|
||||
New service functions (get_or_create_daily_summary, compute_daily_summary,
|
||||
overwrite_daily_summary) wrap get_daily_summary with DB-backed snapshot storage.
|
||||
Each item fails independently; failures do NOT contribute to totals, and the UI
|
||||
must treat any failed item as incomplete accounting data.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import date, datetime, timedelta
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
import pytz
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models.upstream import Upstream
|
||||
from app.models.website import Website
|
||||
from app.services.upstream_client import UpstreamClient
|
||||
@@ -26,7 +30,7 @@ from app.services.website_client import Sub2ApiWebsiteClient, WebsiteError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SHANGHAI = pytz.timezone("Asia/Shanghai")
|
||||
SHANGHAI_TZ = pytz.timezone("Asia/Shanghai")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
@@ -63,13 +67,17 @@ def classify_upstream(upstream: Upstream) -> str:
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
def yesterday_shanghai() -> date:
|
||||
return datetime.now(_SHANGHAI).date() - timedelta(days=1)
|
||||
return datetime.now(SHANGHAI_TZ).date() - timedelta(days=1)
|
||||
|
||||
|
||||
def today_shanghai() -> date:
|
||||
return datetime.now(SHANGHAI_TZ).date()
|
||||
|
||||
|
||||
def date_to_shanghai_timestamps(d: date) -> tuple[int, int]:
|
||||
"""Return (start_unix, end_unix) for 00:00:00..23:59:59 Asia/Shanghai on date d."""
|
||||
start = _SHANGHAI.localize(datetime(d.year, d.month, d.day, 0, 0, 0))
|
||||
end = _SHANGHAI.localize(datetime(d.year, d.month, d.day, 23, 59, 59))
|
||||
start = SHANGHAI_TZ.localize(datetime(d.year, d.month, d.day, 0, 0, 0))
|
||||
end = SHANGHAI_TZ.localize(datetime(d.year, d.month, d.day, 23, 59, 59))
|
||||
return int(start.timestamp()), int(end.timestamp())
|
||||
|
||||
|
||||
@@ -294,3 +302,142 @@ def get_daily_summary(
|
||||
"failed_count": failed_count,
|
||||
"success": failed_count == 0,
|
||||
}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# DB-backed snapshot storage
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
def _truncate_errors(summary: dict[str, Any], max_len: int = 500) -> None:
|
||||
"""Truncate error messages in place to keep summary_json from growing unbounded."""
|
||||
for item in summary.get("website_items", []) + summary.get("upstream_items", []):
|
||||
err = item.get("error")
|
||||
if err and len(err) > max_len:
|
||||
item["error"] = err[:max_len] + "..."
|
||||
|
||||
|
||||
def _save_summary(db: Session, date_key: date, summary: dict[str, Any]) -> datetime:
|
||||
"""Save a computed summary to the database. Returns computed_at."""
|
||||
_truncate_errors(summary)
|
||||
from app.models.finance_daily_summary import FinanceDailySummary
|
||||
now = datetime.now(timezone.utc)
|
||||
row = FinanceDailySummary(
|
||||
stat_date=date_key,
|
||||
summary_json=json.dumps(summary, ensure_ascii=False, default=str),
|
||||
success=summary.get("success", False),
|
||||
computed_at=now,
|
||||
)
|
||||
db.add(row)
|
||||
db.commit()
|
||||
return now
|
||||
|
||||
|
||||
def compute_daily_summary(
|
||||
db: Session,
|
||||
target_date: date,
|
||||
) -> dict[str, Any]:
|
||||
"""Compute a fresh daily summary (no DB write). Used for compare."""
|
||||
websites = db.query(Website).filter(Website.enabled == True).all()
|
||||
upstreams = db.query(Upstream).filter(Upstream.enabled == True).all()
|
||||
return get_daily_summary(websites, upstreams, target_date)
|
||||
|
||||
|
||||
def _load_summary(row: Any) -> dict[str, Any]:
|
||||
"""Load summary dict from a FinanceDailySummary row, adding metadata."""
|
||||
data = json.loads(row.summary_json)
|
||||
data["computed_at"] = row.computed_at.isoformat()
|
||||
data["from_snapshot"] = True
|
||||
return data
|
||||
|
||||
|
||||
def get_or_create_daily_summary(
|
||||
db: Session,
|
||||
target_date: date,
|
||||
) -> dict[str, Any]:
|
||||
"""Return stored summary for target_date, or compute + save if missing.
|
||||
|
||||
If a concurrent request already saved a row between our query and commit,
|
||||
we rollback and return the existing row. This prevents duplicate-key errors.
|
||||
"""
|
||||
from app.models.finance_daily_summary import FinanceDailySummary
|
||||
|
||||
existing = db.query(FinanceDailySummary).filter(
|
||||
FinanceDailySummary.stat_date == target_date
|
||||
).first()
|
||||
if existing:
|
||||
return _load_summary(existing)
|
||||
|
||||
# Compute and save
|
||||
summary = compute_daily_summary(db, target_date)
|
||||
try:
|
||||
now = _save_summary(db, target_date, summary)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
# Race: another request may have inserted this date. Read again.
|
||||
existing = db.query(FinanceDailySummary).filter(
|
||||
FinanceDailySummary.stat_date == target_date
|
||||
).first()
|
||||
if existing:
|
||||
return _load_summary(existing)
|
||||
# Not a race — real error. Re-raise.
|
||||
raise
|
||||
|
||||
summary["computed_at"] = now.isoformat()
|
||||
summary["from_snapshot"] = False
|
||||
return summary
|
||||
|
||||
|
||||
def save_summary_if_absent(
|
||||
db: Session,
|
||||
target_date: date,
|
||||
) -> bool:
|
||||
"""Compute and save summary for target_date if no snapshot exists yet.
|
||||
|
||||
Returns True if a new snapshot was saved, False if one already existed.
|
||||
Safe against concurrent callers: on IntegrityError rolls back and
|
||||
re-checks. Used by the scheduler cron job.
|
||||
"""
|
||||
from app.models.finance_daily_summary import FinanceDailySummary
|
||||
|
||||
existing = db.query(FinanceDailySummary).filter(
|
||||
FinanceDailySummary.stat_date == target_date
|
||||
).first()
|
||||
if existing:
|
||||
return False
|
||||
|
||||
summary = compute_daily_summary(db, target_date)
|
||||
try:
|
||||
_save_summary(db, target_date, summary)
|
||||
return True
|
||||
except Exception:
|
||||
db.rollback()
|
||||
# Race: another caller may have inserted this date.
|
||||
existing = db.query(FinanceDailySummary).filter(
|
||||
FinanceDailySummary.stat_date == target_date
|
||||
).first()
|
||||
if existing:
|
||||
return False
|
||||
raise
|
||||
|
||||
|
||||
def overwrite_daily_summary(
|
||||
db: Session,
|
||||
target_date: date,
|
||||
) -> dict[str, Any]:
|
||||
"""Recompute and overwrite the stored snapshot for target_date.
|
||||
|
||||
Used when the user explicitly requests a refresh.
|
||||
Deletes any existing row for this date first, then inserts the new one.
|
||||
"""
|
||||
from app.models.finance_daily_summary import FinanceDailySummary
|
||||
|
||||
db.query(FinanceDailySummary).filter(
|
||||
FinanceDailySummary.stat_date == target_date
|
||||
).delete()
|
||||
db.flush()
|
||||
|
||||
summary = compute_daily_summary(db, target_date)
|
||||
now = _save_summary(db, target_date, summary)
|
||||
summary["computed_at"] = now.isoformat()
|
||||
summary["from_snapshot"] = False
|
||||
return summary
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""APScheduler background scheduler for upstream checks."""
|
||||
"""APScheduler background scheduler for upstream checks and daily tasks."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
@@ -8,6 +8,7 @@ from typing import Any
|
||||
|
||||
from apscheduler.executors.pool import ThreadPoolExecutor
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import SessionLocal
|
||||
@@ -365,6 +366,16 @@ def start_scheduler() -> None:
|
||||
max_instances=1,
|
||||
misfire_grace_time=3600,
|
||||
)
|
||||
# Daily finance summary at 00:05 Asia/Shanghai
|
||||
_scheduler.add_job(
|
||||
_compute_finance_daily_summary,
|
||||
trigger=CronTrigger(timezone="Asia/Shanghai", hour=0, minute=5),
|
||||
id="finance_daily_summary",
|
||||
replace_existing=True,
|
||||
coalesce=True,
|
||||
max_instances=1,
|
||||
misfire_grace_time=3600,
|
||||
)
|
||||
logger.info("scheduler started with %d upstream job(s)", len(upstreams))
|
||||
finally:
|
||||
db.close()
|
||||
@@ -383,6 +394,28 @@ def _cleanup_external_logs() -> None:
|
||||
db.close()
|
||||
|
||||
|
||||
def _compute_finance_daily_summary() -> None:
|
||||
"""Compute and save finance daily summary for yesterday (Asia/Shanghai).
|
||||
|
||||
Runs daily at 00:05 Asia/Shanghai. Delegates to save_summary_if_absent
|
||||
which handles the skip-if-exists logic and concurrent-save race.
|
||||
"""
|
||||
from app.services.finance_service import yesterday_shanghai, save_summary_if_absent
|
||||
|
||||
target = yesterday_shanghai()
|
||||
db = SessionLocal()
|
||||
try:
|
||||
saved = save_summary_if_absent(db, target)
|
||||
if saved:
|
||||
logger.info("finance daily summary for %s saved", target)
|
||||
else:
|
||||
logger.info("finance daily summary for %s already exists, skipping", target)
|
||||
except Exception:
|
||||
logger.exception("failed to compute finance daily summary for %s", target)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def stop_scheduler() -> None:
|
||||
if _scheduler.running:
|
||||
_scheduler.shutdown(wait=True)
|
||||
|
||||
Reference in New Issue
Block a user