diff --git a/backend/README.md b/backend/README.md index 16591a95..71ce03cb 100644 --- a/backend/README.md +++ b/backend/README.md @@ -32,9 +32,17 @@ uv run ruff check . ## Gold Cache and Notifications - Application startup checks both realtime and official gold caches, then checks - them every minute. Data older than 30 minutes refreshes without HTTP traffic. + them at wall-clock minute boundaries. Each half-hour batch starts 5 minutes + early: 01:25 collects the 01:30 batch, 01:55 collects the 02:00 batch, without + HTTP traffic. All regions in a refresh share the batch timestamp selected at + the start, even if the work crosses a half-hour boundary. Existing public + `updataTime` types remain number for realtime and string for official. Redis locks prevent duplicate work across workers. Realtime region pages stay 8 seconds apart. Failures retain the last successful data for up to 24 hours. +- Realtime uses the arithmetic mean of displayed unit prices from rows 4–10 + (`i.item1`, slice `[3:10]`), in 元/万金币, rounded to 2 decimals. This replaces + the earlier 万金币/元 average and uses a new Redis namespace so old units are + never returned as new prices. No BeautifulSoup or browser dependency is added. - Official authentication uses `HELPER_TOKEN` and `HELPER_ID`. Authentication failures (`returnCode=-30003`, explicit token-expiry messages or HTTP 401) can notify via Bark. Network errors do not trigger token-expiry notifications. @@ -46,8 +54,9 @@ uv run ruff check . included in notification text or logged push errors. - Official gold responses also contain `max` and `min`: the largest/smallest values of `10000 / price` across both rank arrays, in 万金币 per 100点券, - rounded to 4 decimals. Nonpositive/invalid prices are excluded; absent valid - quotes yield `null`. Existing rank arrays retain their 点券 per 100万金币 unit. + rounded to 2 decimals. Nonpositive/invalid prices are excluded; absent valid + quotes yield `null`. Rank arrays retain their 点券 per 100万金币 unit and are + also rounded to 2 decimals. JSON numbers do not force trailing zeroes. ## Compatibility Rules diff --git a/backend/api/routes/colg.py b/backend/api/routes/colg.py index c721e7f1..1782b550 100644 --- a/backend/api/routes/colg.py +++ b/backend/api/routes/colg.py @@ -41,9 +41,9 @@ def get_date_str(timestamp: int) -> str: @router.get('/gold/realtime/', operation_id='realtimeGold', response_model=Response[list[RealTimeGold]]) async def get_real_time_gold(redis: Annotated[AsyncRedis, Depends(thirdpart_gold.get_redis)], background_tasks: BackgroundTasks) -> Response[list[RealTimeGold]]: - """获取网站金币比例，数据为各跨区首次报价前200条的平均万金币/元。 + """获取网站金币单价，取页面第4～10条单价均值，元/万金币，两位小数。 - updataTime为数字秒级时间戳，code为跨区标识。与thirdpart接口共用Redis缓存和 + updataTime为数据所属半小时刻度的数字秒级时间戳，:25/:55提前采集对应:30/:00数据。与thirdpart接口共用Redis缓存和 后台刷新任务：30分钟自动更新，也支持按需补齐，逐区间隔8秒；初始化未齐时响应体code=202。 """ return await get_thirdpart_gold(redis, background_tasks) @@ -145,11 +145,11 @@ async def get_official_gold(redis: Annotated[AsyncRedis, Depends(thirdpart_gold. @router.get('/gold/thirdpart/', operation_id='thirdpartGold', response_model=Response[list[RealTimeGold]]) async def get_thirdpart_gold(redis: Annotated[AsyncRedis, Depends(thirdpart_gold.get_redis)], background_tasks: BackgroundTasks) -> Response[list[RealTimeGold]]: - """获取跨1～跨8（含跨3A/3B）的第三方首批报价均值，单位：万金币/元。 + """获取跨1～跨8（含跨3A/3B）的第三方单价均值，单位：元/万金币。 - 不限制最低总价；各平台首次返回的有效报价汇总、去重、按单价取前200条， - 对每条报价的1/Price取算术平均，不按库存或份数加权。保留上游原始价格， - 不额外计算交易手续费。该接口无上游分页参数，前200条是本接口的样本口径。 + 不额外限制最低总价；汇总渠道报价并按公开页面展示规则排序、去重， + 对i.item1的第4～10条（Python切片[3:10]）显示单价取算术平均， + 四舍五入至两位小数，不按库存或份数加权。 每30分钟自动后台刷新，跨区页面最短间隔8秒（每分钟不超过8次），Redis锁避免重复刷新。 初始化跨区未齐时响应体code=202并注明进度；9区缓存齐全才返回code=200。 diff --git a/backend/api/types/colg.py b/backend/api/types/colg.py index 78cca932..7672ac87 100644 --- a/backend/api/types/colg.py +++ b/backend/api/types/colg.py @@ -1,4 +1,6 @@ -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator + +from api.utils.gold_time import half_hour_start GOLD_REGION_CODES = { @@ -8,20 +10,29 @@ GOLD_REGION_CODES = { class RealTimeGold(BaseModel): - updataTime: int = Field(..., ge=0, description='更新时间，Unix秒级时间戳') + updataTime: int = Field(..., ge=0, description='数据所属半小时刻度的Unix秒级时间戳，提前5分钟开始采集') """更新时间""" region: str = Field(..., description='跨区') """跨区""" code: str = Field(..., description='跨区code：k1、k2、k3a、k3b、k4、k5、k6、k7、k8') - goldPrice: float = Field(..., gt=0, allow_inf_nan=False, description='金币比例，1元对应的万金币数量') + goldPrice: float = Field(..., ge=0, allow_inf_nan=False, description='页面第4～10条单价均值，元/万金币，四舍五入至两位小数') """金价""" + @field_validator('updataTime') + @classmethod + def align_update_time(cls, value): + return half_hour_start(value) + + @field_validator('goldPrice') + @classmethod + def round_gold_price(cls, value): + return round(value, 2) + class ThirdPartGoldCache(RealTimeGold): """内部缓存数据，不作为接口返回模型。""" - goldPrice: float = Field(..., gt=0, allow_inf_nan=False, description='首页有效报价换算为万金币/元后的算术平均值') - sampleCount: int = Field(..., ge=1, le=200, description='参与均值计算的报价条数') + sampleCount: int = Field(..., ge=1, le=7, description='第4～10条中参与均值计算的报价条数') stale: bool = Field(..., description='数据是否已超过30分钟刷新周期') @@ -41,7 +52,7 @@ class HistoryGold(BaseModel): """echarts数据""" class OfficialGold(BaseModel): - updataTime: str = Field(..., pattern=r'^\d+$', description='更新时间，Unix秒级时间戳字符串') + updataTime: str = Field(..., pattern=r'^\d+$', description='数据所属半小时刻度的Unix秒级时间戳字符串，提前5分钟开始采集') """更新时间""" region: str = Field(..., description='跨区') """跨区""" @@ -52,3 +63,18 @@ class OfficialGold(BaseModel): """金币寄售最高成交价排名前5的价格""" max: float | None = Field(..., description='100点券可兑换的最大万金币数量，无有效报价时为null') min: float | None = Field(..., description='100点券可兑换的最小万金币数量，无有效报价时为null') + + @field_validator('updataTime') + @classmethod + def align_update_time(cls, value): + return str(half_hour_start(int(value))) + + @field_validator('goldDealRank', 'goldTopDealRank') + @classmethod + def round_ranks(cls, values): + return [round(value, 2) for value in values] + + @field_validator('max', 'min') + @classmethod + def round_ratios(cls, value): + return round(value, 2) if value is not None else None diff --git a/backend/api/utils/gold_refresh.py b/backend/api/utils/gold_refresh.py index 6e765335..b2501da2 100644 --- a/backend/api/utils/gold_refresh.py +++ b/backend/api/utils/gold_refresh.py @@ -2,6 +2,7 @@ import asyncio import logging +import time from api.utils import official_gold, thirdpart_gold @@ -27,7 +28,8 @@ async def _periodic(job): await job() except Exception: logger.exception('periodic gold refresh check failed job=%s', job.__name__) - await asyncio.sleep(CHECK_INTERVAL_SECONDS) + # 对齐墙钟分钟边界，:25/:55检测到下一半小时数据周期并提前采集。 + await asyncio.sleep(CHECK_INTERVAL_SECONDS - time.time() % CHECK_INTERVAL_SECONDS) async def start_periodic_refresh(): diff --git a/backend/api/utils/gold_time.py b/backend/api/utils/gold_time.py new file mode 100644 index 00000000..f1a2ffc9 --- /dev/null +++ b/backend/api/utils/gold_time.py @@ -0,0 +1,18 @@ +"""每个半小时刻度提前5分钟采集，返回值标记对应的半小时刻度。""" + +import time + +HALF_HOUR_SECONDS = 30 * 60 +UPDATE_LEAD_SECONDS = 5 * 60 + + +def half_hour_start(timestamp: float | None = None) -> int: + if timestamp is None: + timestamp = time.time() + return int(timestamp) // HALF_HOUR_SECONDS * HALF_HOUR_SECONDS + + +def refresh_slot(timestamp: float | None = None) -> int: + if timestamp is None: + timestamp = time.time() + return half_hour_start(timestamp + UPDATE_LEAD_SECONDS) diff --git a/backend/api/utils/official_gold.py b/backend/api/utils/official_gold.py index 0faefdff..f2b70103 100644 --- a/backend/api/utils/official_gold.py +++ b/backend/api/utils/official_gold.py @@ -1,7 +1,6 @@ """官方寄售缓存、认证失败识别及可选BARK通知。""" import asyncio -from datetime import datetime, timezone import hashlib import json import logging @@ -14,6 +13,7 @@ import aiohttp from api.core.Gzip import gzip_unzip, gzip_zip from api.types.colg import GOLD_REGION_CODES, OfficialGold from api.utils.thirdpart_gold import RELEASE_LOCK +from api.utils.gold_time import UPDATE_LEAD_SECONDS, half_hour_start, refresh_slot from config.main import config logger = logging.getLogger('dcalc.official_gold') @@ -43,14 +43,14 @@ def normalize_row(item: dict) -> dict: except (TypeError, ValueError, OverflowError): continue if math.isfinite(price): - prices.append(price) + prices.append(round(price, 2)) ranks[name] = prices ratios = [10000 / price for prices in ranks.values() for price in prices if price > 0 and math.isfinite(10000 / price)] return OfficialGold.model_validate({ - **item, **ranks, - 'max': round(max(ratios), 4) if ratios else None, - 'min': round(min(ratios), 4) if ratios else None, + **item, **ranks, 'updataTime': str(half_hour_start(int(item['updataTime']))), + 'max': round(max(ratios), 2) if ratios else None, + 'min': round(min(ratios), 2) if ratios else None, }).model_dump() @@ -64,7 +64,10 @@ def is_auth_failure(payload: dict) -> bool: )) -async def fetch_gold() -> list[dict]: +async def fetch_gold(*, started_at: float | None = None) -> list[dict]: + if started_at is None: + started_at = time.time() + update_time = str(refresh_slot(started_at)) timeout = aiohttp.ClientTimeout(total=10) async with aiohttp.ClientSession(timeout=timeout) as session: async def fetch_region(region): @@ -81,7 +84,7 @@ async def fetch_gold() -> list[dict]: resp.raise_for_status() data = payload['data'] return normalize_row({ - 'updataTime': str(int(datetime.now(timezone.utc).timestamp())), + 'updataTime': update_time, 'region': f'跨{region}', 'code': GOLD_REGION_CODES[f'跨{region}'], 'goldDealRank': [float(item['price']) for item in data['godDealRank'][:5]], 'goldTopDealRank': [float(item['price']) for item in data['godTopDealRank'][:5]], @@ -150,7 +153,7 @@ async def read_cache(redis) -> list[dict]: for item in json.loads(gzip_unzip(value))] if [row['region'] for row in rows] != [f'跨{region}' for region in REGIONS]: return [] - if any(not 0 <= time.time() - int(row['updataTime']) < RETENTION_SECONDS for row in rows): + if any(not -UPDATE_LEAD_SECONDS <= time.time() - int(row['updataTime']) < RETENTION_SECONDS for row in rows): return [] return rows except (ValueError, TypeError, KeyError, OSError): @@ -160,7 +163,7 @@ async def read_cache(redis) -> list[dict]: def is_fresh(rows: list[dict]) -> bool: return (len(rows) == len(REGIONS) - and all(0 <= time.time() - int(row['updataTime']) < FRESH_SECONDS for row in rows)) + and all(int(row['updataTime']) == refresh_slot() for row in rows)) async def refresh_cache(redis) -> bool: @@ -173,7 +176,7 @@ async def refresh_cache(redis) -> bool: if is_fresh(await read_cache(redis)) or await redis.get(COOLDOWN_KEY): return False try: - data = await fetch_gold() + data = await fetch_gold(started_at=time.time()) except TokenExpired: logger.warning('official gold authentication failed; check HELPER_TOKEN and HELPER_ID') await notify_token_expired(redis) diff --git a/backend/api/utils/thirdpart_gold.py b/backend/api/utils/thirdpart_gold.py index ab1e8817..bea6d61b 100644 --- a/backend/api/utils/thirdpart_gold.py +++ b/backend/api/utils/thirdpart_gold.py @@ -9,7 +9,6 @@ import math import re from statistics import fmean import time -from datetime import datetime, timedelta, timezone from urllib.parse import urljoin, urlsplit from uuid import uuid4 @@ -19,6 +18,7 @@ from redis.asyncio.retry import Retry from redis.backoff import NoBackoff from api.types.colg import GOLD_REGION_CODES, ThirdPartGoldCache +from api.utils.gold_time import refresh_slot from config.main import config logger = logging.getLogger('dcalc.thirdpart_gold') @@ -36,6 +36,7 @@ HEADERS = { # 网站公开 DNF 页面已停用/移除的渠道；不请求 VIP 专属渠道。 DISABLED_PLATFORMS = {1, 12, 21, 50, 51, 52, 53} PAGE_SIZE = 200 +SAMPLE_SLICE = slice(3, 10) FRESH_SECONDS = 30 * 60 RETENTION_SECONDS = 24 * 60 * 60 # 按每分钟最多8次页面访问处理，理论间隔7.5秒，取8秒留出边界余量。 @@ -44,7 +45,8 @@ REGION_INTERVAL = 8 LOCK_SECONDS = 15 * 60 ERROR_COOLDOWN_SECONDS = 5 * 60 REDIS_TIMEOUT_SECONDS = 1 -CACHE_PREFIX = 'openapi:colg:thirdpart_gold:v1' +# 统计范围和单位已改变，旧缓存的万金币/元均值不能作为新的元/万金币单价返回。 +CACHE_PREFIX = 'openapi:colg:thirdpart_gold:v2' LOCK_KEY = f'{CACHE_PREFIX}:lock' COOLDOWN_KEY = f'{CACHE_PREFIX}:cooldown' RELEASE_LOCK = """ @@ -126,7 +128,7 @@ def make_payload(channel: dict, seed: dict, user: dict) -> dict: def first_page_average(offers: list[dict]) -> tuple[float, int]: - """等权平均每条报价的 1/Price（万金币/元），不是 1/平均单价。""" + """匹配公开页面 i.item1 的第4～10条，平均显示单价（元/万金币）。""" valid = [] for offer in offers: try: @@ -135,9 +137,12 @@ def first_page_average(offers: list[dict]) -> tuple[float, int]: number = float(offer['Number']) except (KeyError, TypeError, ValueError, OverflowError): continue - if not all(math.isfinite(value) and value > 0 for value in (price, money, number)): + if not all(math.isfinite(value) for value in (price, money, number)) or price < 0: continue - if not math.isfinite(1 / price): + # 页面默认展示条件，不额外设置price1/price2价格过滤。 + if money != -1 and (money * number < 5 or price == 0): + continue + if money >= 1_000_000 or (offer.get('Seller') or '').startswith('智莹科技'): continue valid.append({**offer, 'Price': price}) valid.sort(key=lambda item: (item['Price'], item['PfId'], item.get('BuyUrl') or '')) @@ -152,9 +157,12 @@ def first_page_average(offers: list[dict]) -> tuple[float, int]: page.append(offer) if len(page) == PAGE_SIZE: break - if not page: + selected = page[SAMPLE_SLICE] + if not selected: raise ValueError('YXDR 未返回有效报价') - return round(fmean(1 / item['Price'] for item in page), 4), len(page) + # 页面 Math.floorDigits(price, 5) 先toFixed(6)再删掉末位。 + prices = [float(f"{item['Price']:.6f}"[:-1]) for item in selected] + return round(fmean(prices), 2), len(prices) async def _read_response(resp) -> str: @@ -165,7 +173,9 @@ async def _read_response(resp) -> str: return text -async def fetch_region(session: aiohttp.ClientSession, region: str) -> dict: +async def fetch_region(session: aiohttp.ClientSession, region: str, *, started_at: float | None = None) -> dict: + if started_at is None: + started_at = time.time() page_url = f'{BASE_URL}/bijiaqi/dnf/youxibi/{region}' async with session.get(page_url) as resp: page = await _read_response(resp) @@ -210,10 +220,9 @@ async def fetch_region(session: aiohttp.ClientSession, region: str) -> dict: if isinstance(result, BaseException): raise result price, count = first_page_average([offer for result in results for offer in result]) - now = datetime.now(timezone(timedelta(hours=8))) return {'region': REGIONS[region], 'code': GOLD_REGION_CODES[REGIONS[region]], 'goldPrice': price, 'sampleCount': count, - 'updataTime': int(now.timestamp()), 'fetchedAt': now.timestamp()} + 'updataTime': refresh_slot(started_at), 'fetchedAt': started_at} async def read_cache(redis) -> list[dict]: @@ -229,8 +238,9 @@ async def read_cache(redis) -> list[dict]: continue # 兼容此前缓存的格式化日期及缺少code的记录，无须重新抓取所有跨区。 result.append(ThirdPartGoldCache.model_validate({ - **item, 'updataTime': int(item['fetchedAt']), - 'code': GOLD_REGION_CODES[REGIONS[region]], 'stale': age >= FRESH_SECONDS, + **item, 'updataTime': refresh_slot(item['fetchedAt']), + 'code': GOLD_REGION_CODES[REGIONS[region]], + 'stale': refresh_slot(item['fetchedAt']) < refresh_slot(), }).model_dump()) except (KeyError, TypeError, ValueError): logger.warning('invalid thirdpart gold cache region=%s', region) @@ -270,6 +280,7 @@ async def refresh_cache(token: str): fresh_regions = {item['region'] for item in cached if not item['stale']} pending = [region for region, name in REGIONS.items() if name not in fresh_regions] successful, last_start = 0, None + started_at = time.time() timeout = aiohttp.ClientTimeout(total=15) async with aiohttp.ClientSession(headers=HEADERS, timeout=timeout, connector=aiohttp.TCPConnector(limit=5)) as session: @@ -281,7 +292,7 @@ async def refresh_cache(token: str): return last_start = time.monotonic() try: - item = await fetch_region(session, region) + item = await fetch_region(session, region, started_at=started_at) if await redis.get(LOCK_KEY) not in (token, token.encode()): return await redis.set(f'{CACHE_PREFIX}:{region}', json.dumps(item), diff --git a/backend/tests/unit/test_colg_official.py b/backend/tests/unit/test_colg_official.py index 2b1f0df5..97d3e543 100644 --- a/backend/tests/unit/test_colg_official.py +++ b/backend/tests/unit/test_colg_official.py @@ -6,6 +6,7 @@ import pytest from api.utils import official_gold as gold from api.core.Gzip import gzip_zip +from api.utils.gold_time import refresh_slot class FakeResponse: @@ -67,14 +68,37 @@ def test_fetch_official_gold_uses_async_http_with_isolated_forms(monkeypatch): assert [item['region'] for item in result] == [f'跨{region}' for region in gold.REGIONS] assert [item['code'] for item in result] == ['k1', 'k2', 'k3a', 'k3b', 'k4', 'k5', 'k6', 'k7'] assert all(isinstance(item['updataTime'], str) and item['updataTime'].isdigit() for item in result) - assert abs(int(result[0]['updataTime']) - time.time()) < 2 + assert int(result[0]['updataTime']) == refresh_slot() assert result[0]['goldDealRank'] == [100.5, 101, 102, 103, 104] assert result[0]['goldTopDealRank'] == [200.25, 201, 202, 203, 204] - assert result[0]['max'] == round(10000 / 100.5, 4) - assert result[0]['min'] == round(10000 / 204, 4) + assert result[0]['max'] == round(10000 / 100.5, 2) + assert result[0]['min'] == round(10000 / 204, 2) assert sessions[0].kwargs['timeout'].total == 10 +def test_official_batch_timestamp_comes_from_start_not_fetch_completion(monkeypatch): + started_at = 1793766300 # 与真实结束时间不同的固定开始时间。 + monkeypatch.setattr(gold.aiohttp, 'ClientSession', lambda **kwargs: FakeSession([], **kwargs)) + rows = asyncio.run(gold.fetch_gold(started_at=started_at)) + assert {row['updataTime'] for row in rows} == {str(refresh_slot(started_at))} + + +def test_preloaded_next_slot_cache_is_servable_before_the_half_hour(monkeypatch): + from datetime import datetime, timedelta, timezone + china = timezone(timedelta(hours=8)) + now = datetime(2026, 11, 4, 1, 25, tzinfo=china).timestamp() + monkeypatch.setattr(gold.time, 'time', lambda: now) + rows = cached_rows() + redis = FakeRedis() + redis.values[gold.CACHE_KEY] = gzip_zip(json.dumps(rows)) + assert gold.is_fresh(asyncio.run(gold.read_cache(redis))) + target = int(rows[0]['updataTime']) + assert target > now + assert target - now == 300 + monkeypatch.setattr(gold.time, 'time', lambda: now + 1800) + assert not gold.is_fresh(rows) + + def test_official_route_uses_new_cache_contract_and_numeric_rank_arrays(monkeypatch, client): calls = [] @@ -82,7 +106,7 @@ def test_official_route_uses_new_cache_contract_and_numeric_rank_arrays(monkeypa calls.append(redis) return [{'updataTime': '1793711716', 'region': '跨3B', 'code': 'k3b', 'goldDealRank': [100.5, 101], 'goldTopDealRank': [200.25, 201], - 'max': round(10000 / 100.5, 4), 'min': round(10000 / 201, 4)}] + 'max': round(10000 / 100.5, 2), 'min': round(10000 / 201, 2)}] monkeypatch.setattr(gold, 'get_data', cached) body = client.get('/api/open/gold/official/').json() @@ -95,8 +119,8 @@ def test_official_route_uses_new_cache_contract_and_numeric_rank_arrays(monkeypa assert row['code'] == 'k3b' assert row['goldDealRank'] == [100.5, 101] assert row['goldTopDealRank'] == [200.25, 201] - assert row['max'] == round(10000 / 100.5, 4) - assert row['min'] == round(10000 / 201, 4) + assert row['max'] == round(10000 / 100.5, 2) + assert row['min'] == round(10000 / 201, 2) for rank in (row['goldDealRank'], row['goldTopDealRank']): assert len(rank) <= 5 assert all(isinstance(value, (int, float)) for value in rank) @@ -140,7 +164,7 @@ class FakeRedis: def cached_rows(age=0): - return [{'updataTime': str(int(time.time() - age)), 'region': f'跨{region}', + return [{'updataTime': str(refresh_slot(time.time() - age)), 'region': f'跨{region}', 'code': f'k{region.lower()}', 'goldDealRank': [100.5], 'goldTopDealRank': [200.25]} for region in gold.REGIONS] @@ -169,8 +193,8 @@ def test_old_official_cache_gets_max_and_min_without_refetch(): redis.values[gold.CACHE_KEY] = gzip_zip(json.dumps(cached_rows())) rows = asyncio.run(gold.read_cache(redis)) assert len(rows) == 8 - assert rows[0]['max'] == round(10000 / 100.5, 4) - assert rows[0]['min'] == round(10000 / 200.25, 4) + assert rows[0]['max'] == round(10000 / 100.5, 2) + assert rows[0]['min'] == round(10000 / 200.25, 2) @pytest.mark.parametrize('payload', [ @@ -206,7 +230,7 @@ def test_refresh_updates_cache_without_http_request_and_skips_when_fresh(monkeyp redis = FakeRedis() calls = [] - async def fetch(): + async def fetch(*, started_at=None): calls.append(1) return cached_rows() @@ -225,7 +249,7 @@ def test_failed_refresh_preserves_cache_and_notifies_once_per_fetch(monkeypatch) redis.values[gold.CACHE_KEY] = old alerts = [] - async def fetch(): + async def fetch(*, started_at=None): raise gold.TokenExpired() async def notify(redis): @@ -247,7 +271,7 @@ def test_network_failure_preserves_cache_without_token_alert(monkeypatch): old = gzip_zip(json.dumps(cached_rows(age=1900))) redis.values[gold.CACHE_KEY] = old - async def fetch(): + async def fetch(*, started_at=None): raise TimeoutError('upstream unavailable') async def notify(redis): diff --git a/backend/tests/unit/test_gold_time.py b/backend/tests/unit/test_gold_time.py new file mode 100644 index 00000000..5b8d0c59 --- /dev/null +++ b/backend/tests/unit/test_gold_time.py @@ -0,0 +1,38 @@ +from datetime import datetime, timedelta, timezone + +import pytest + +from api.types.colg import OfficialGold, RealTimeGold +from api.utils.gold_time import half_hour_start, refresh_slot + +CHINA = timezone(timedelta(hours=8)) + + +def stamp(hour, minute, second=0): + return int(datetime(2026, 11, 4, hour, minute, second, tzinfo=CHINA).timestamp()) + + +@pytest.mark.parametrize(('hour', 'minute', 'second', 'target_hour', 'target_minute'), [ + (1, 24, 59, 1, 0), (1, 25, 0, 1, 30), (1, 29, 59, 1, 30), + (1, 30, 0, 1, 30), (1, 54, 59, 1, 30), (1, 55, 0, 2, 0), (2, 0, 0, 2, 0), + (23, 55, 0, 0, 0), +]) +def test_refresh_slot_advances_five_minutes_before_boundary(hour, minute, second, target_hour, target_minute): + expected = stamp(target_hour, target_minute) + if hour == 23: + expected += 24 * 60 * 60 + assert refresh_slot(stamp(hour, minute, second)) == expected + + +def test_response_types_round_price_values_and_keep_timestamp_types(): + realtime = RealTimeGold(updataTime=stamp(1, 30), region='跨1', code='k1', goldPrice=.016789) + assert realtime.goldPrice == .02 + assert realtime.updataTime == stamp(1, 30) + assert isinstance(realtime.updataTime, int) + official = OfficialGold(updataTime=str(stamp(1, 30)), region='跨1', code='k1', + goldDealRank=[125.556], goldTopDealRank=[250.123], max=79.64444, min=39.9876) + assert official.goldDealRank == [125.56] + assert official.goldTopDealRank == [250.12] + assert official.max == 79.64 and official.min == 39.99 + assert official.updataTime == str(stamp(1, 30)) + assert half_hour_start(stamp(1, 29, 59)) == stamp(1, 0) diff --git a/backend/tests/unit/test_thirdpart_gold.py b/backend/tests/unit/test_thirdpart_gold.py index 60fd1c28..dcf674bf 100644 --- a/backend/tests/unit/test_thirdpart_gold.py +++ b/backend/tests/unit/test_thirdpart_gold.py @@ -9,16 +9,17 @@ from fastapi import BackgroundTasks from api.routes import colg from api.types.colg import GOLD_REGION_CODES from api.utils import thirdpart_gold as gold +from api.utils.gold_time import refresh_slot -def offer(price, url='listing', platform=17, money=1): +def offer(price, url='listing', platform=17, money=10): return {'Price': price, 'Money': money, 'Number': 1, 'PfId': platform, 'BuyUrl': url} def cached_item(region='跨1', age=0): fetched_at = time.time() - age - return {'region': region, 'goldPrice': 100.0, 'sampleCount': 10, - 'code': GOLD_REGION_CODES[region], 'updataTime': int(fetched_at), + return {'region': region, 'goldPrice': .01, 'sampleCount': 7, + 'code': GOLD_REGION_CODES[region], 'updataTime': refresh_slot(fetched_at), 'fetchedAt': fetched_at} @@ -53,22 +54,28 @@ class FakeRedis: return 0 -def test_average_is_mean_of_ratios_without_minimum_money_filter(): - # 100 and 50 万/元 => 75; inverse mean price would incorrectly give 66.67. - assert gold.first_page_average([offer(.01), offer(.02)]) == (75.0, 2) +def test_average_uses_displayed_unit_prices_at_indices_three_through_nine(): + rows = [offer(i / 100) for i in range(1, 12)] + assert gold.first_page_average(rows) == (.07, 7) invalid = [offer(0), offer(-1), offer(float('nan')), offer(float('inf')), offer(.01, money=-1), offer(None), offer(.01, money=float('nan'))] - assert gold.first_page_average(invalid + [offer(.01, money=.5)]) == (100.0, 1) + # 页面允许Money=-1的库存占位报价；另行排除不显示的低额订单。 + invalid.pop(4) + assert gold.first_page_average(invalid + [offer(.01, money=.5)] + rows) == (.07, 7) + assert gold.first_page_average([offer(.016789)] * 10) == (.02, 7) + assert gold.first_page_average(rows[:4]) == (.04, 1) with pytest.raises(ValueError, match='有效报价'): gold.first_page_average(invalid) + with pytest.raises(ValueError, match='有效报价'): + gold.first_page_average(rows[:3]) def test_page_deduplicates_before_cutoff_and_preserves_7881_variants(): rows = [offer(.01, 'duplicate', 40), offer(.01, 'duplicate', 40)] - rows += [offer(.02, str(i), 17) for i in range(199)] - rows += [offer(.5, 'outside', 17)] - assert gold.first_page_average(rows) == (50.25, 200) - assert gold.first_page_average([offer(.01, platform=44), offer(.02, platform=44)]) == (75.0, 2) + rows += [offer(i / 100, str(i), 17) for i in range(2, 12)] + assert gold.first_page_average(rows) == (.07, 7) + rows = [offer(.01, platform=44)] * 3 + [offer(.04, platform=44)] * 7 + assert gold.first_page_average(rows) == (.04, 7) def test_request_signature_matches_observed_browser_request(): @@ -118,14 +125,20 @@ def test_region_fetches_its_own_seed_config_and_all_initial_channels(): def post(self, url, json, **kwargs): posts.append((url, json)) - return FakeResponse('{"success":1,"data":[{"Price":0.01,"Money":1,"Number":1}]}') + return FakeResponse('{"success":1,"data":' + json_module.dumps([ + {'Price': .01, 'Money': 10, 'Number': 1} for _ in range(10) + ]) + '}') - result = asyncio.run(gold.fetch_region(Session(), 'kua8')) + json_module = json + started_at = 1793766590 # 固定开始时间，与响应读取完成的当前时间不同。 + result = asyncio.run(gold.fetch_region(Session(), 'kua8', started_at=started_at)) assert result['region'] == '跨8' assert result['code'] == 'k8' assert isinstance(result['updataTime'], int) - assert result['goldPrice'] == 100 - assert result['sampleCount'] == 1 + assert result['goldPrice'] == .01 + assert result['sampleCount'] == 7 + assert result['updataTime'] == refresh_slot(started_at) + assert result['fetchedAt'] == started_at assert gets == [gold.BASE_URL + '/bijiaqi/dnf/youxibi/kua8', gold.BASE_URL + '/cate/1838/CoinSale_yxb_254_0.js?ver=935'] assert posts[0][1]['secret'] == hashlib.md5(b'fresh-seed_17_config-sign').hexdigest() @@ -148,9 +161,25 @@ def test_cache_omits_missing_regions_marks_old_values_and_retains_order(): assert [row['stale'] for row in rows] == [True, False] +def test_realtime_prefetches_at_25_and_remains_fresh_at_30(monkeypatch): + base = 1793764800 // 1800 * 1800 + clock = {'now': base + 25 * 60} + monkeypatch.setattr(gold.time, 'time', lambda: clock['now']) + redis = FakeRedis() + old = cached_item(age=25 * 60) + redis.values[f'{gold.CACHE_PREFIX}:kua1'] = json.dumps(old) + assert asyncio.run(gold.read_cache(redis))[0]['stale'] is True + redis.values[f'{gold.CACHE_PREFIX}:kua1'] = json.dumps(cached_item()) + prefetched = asyncio.run(gold.read_cache(redis))[0] + assert prefetched['updataTime'] == base + 30 * 60 + assert prefetched['stale'] is False + clock['now'] = base + 30 * 60 + assert asyncio.run(gold.read_cache(redis))[0]['stale'] is False + + def test_invalid_cached_average_is_not_served(): redis = FakeRedis() - redis.values[f'{gold.CACHE_PREFIX}:kua1'] = json.dumps({**cached_item(), 'goldPrice': 0}) + redis.values[f'{gold.CACHE_PREFIX}:kua1'] = json.dumps({**cached_item(), 'goldPrice': -1}) redis.values[f'{gold.CACHE_PREFIX}:kua8'] = json.dumps({**cached_item('跨8'), 'sampleCount': 0}) assert asyncio.run(gold.read_cache(redis)) == [] @@ -164,7 +193,7 @@ def test_legacy_cache_gets_epoch_and_region_code_without_refetching(): rows = asyncio.run(gold.read_cache(redis)) assert len(rows) == 1 assert rows[0]['code'] == 'k3a' - assert rows[0]['updataTime'] == int(legacy['fetchedAt']) + assert rows[0]['updataTime'] == refresh_slot(legacy['fetchedAt']) assert not redis.writes @@ -203,7 +232,7 @@ def test_refresh_preserves_failed_region_and_releases_its_lock(monkeypatch, limi monkeypatch.setattr(gold, 'REGION_INTERVAL', 0) calls = [] - async def fetch(session, region): + async def fetch(session, region, *, started_at=None): calls.append(region) if region == 'kua1': raise gold.UpstreamLimited() if limited else ValueError('channel failed') @@ -357,7 +386,7 @@ def test_shutdown_cancels_refresh_and_releases_redis_lock(monkeypatch): async def run(): started = asyncio.Event() - async def fetch(session, region): + async def fetch(session, region, *, started_at=None): started.set() await asyncio.Event().wait() @@ -382,7 +411,7 @@ def test_retry_only_fetches_missing_regions_instead_of_repeating_fresh_pages(mon monkeypatch.setattr(gold, 'create_redis', lambda: redis) calls = [] - async def fetch(session, region): + async def fetch(session, region, *, started_at=None): calls.append(region) return cached_item(gold.REGIONS[region]) @@ -392,6 +421,31 @@ def test_retry_only_fetches_missing_regions_instead_of_repeating_fresh_pages(mon assert len(asyncio.run(gold.read_cache(redis))) == 9 +def test_realtime_regions_share_batch_start_even_when_fetch_crosses_slot_boundary(monkeypatch): + clock = {'now': 1793766590.0} + started_at = clock['now'] + redis = FakeRedis() + redis.values[gold.LOCK_KEY] = 'owner' + monkeypatch.setattr(gold, 'create_redis', lambda: redis) + monkeypatch.setattr(gold, 'REGIONS', {'kua1': '跨1', 'kua8': '跨8'}) + monkeypatch.setattr(gold, 'REGION_INTERVAL', 0) + monkeypatch.setattr(gold.time, 'time', lambda: clock['now']) + starts = [] + + async def fetch(session, region, *, started_at): + starts.append(started_at) + clock['now'] += 1800 + return {**cached_item(gold.REGIONS[region]), 'fetchedAt': started_at, + 'updataTime': refresh_slot(started_at)} + + monkeypatch.setattr(gold, 'fetch_region', fetch) + asyncio.run(gold.refresh_cache('owner')) + assert starts == [started_at, started_at] + timestamps = [json.loads(redis.values[f'{gold.CACHE_PREFIX}:{region}'])['updataTime'] + for region in gold.REGIONS] + assert timestamps == [refresh_slot(started_at)] * 2 + + def test_unresponsive_redis_returns_prompt_error_instead_of_hanging(monkeypatch): async def run(): connections = [] diff --git a/docs/image-cors.md b/docs/image-cors.md new file mode 100644 index 00000000..5c41019e --- /dev/null +++ b/docs/image-cors.md @@ -0,0 +1,87 @@ +# 技能加点图片导出的跨域配置 + +技能树使用 `https://image.dnftools.com` 上的图标。普通 `` 展示不要求 CORS，但 `html-to-image` 导出时需要 fetch 图标并内嵌到图片，因此图床必须返回允许跨域读取的响应头。前端不需要改动，`no-cors` 无法读取图片内容。 + +## 允许范围 + +仅允许 HTTP / HTTPS 的 `*.dnftools.com` 来源，包括多级子域和带端口的来源。例如 `https://dcalc.dnftools.com`、`https://test.dcalc.dnftools.com` 和 `http://dev.dnftools.com:5173`。 + +不包括裸域 `https://dnftools.com`、`localhost`、`null`、其他域名或 `https://dnftools.com.example.org` 这样的伪装后缀。CORS 响应头不能写成 `*.dnftools.com`，应校验 Origin 后返回该完整 Origin，并设置 `Vary: Origin` 让缓存区分来源。 + +## Nginx / OpenResty 图床 + +下面的配置适用于直接编辑站点 `server { ... }` 的环境，不需要在外层 http 中声明 map。 + +### 1. 在 server 层增加来源校验和默认响应头 + +放在 `server_name image.dnftools.com;` 后面，保留其余配置和原有的 HSTS 头： + +```nginx +set $image_cors_origin ""; +if ($http_origin ~* "^https?://([a-z0-9-]+\.)+dnftools\.com(:[0-9]+)?$") { + set $image_cors_origin $http_origin; +} + +add_header Access-Control-Allow-Origin $image_cors_origin always; +add_header Vary "Origin" always; +``` + +未匹配的来源对应空值，Nginx 不输出 Access-Control-Allow-Origin。现有 `location /` 没有自己的 add_header，会继承这里的响应头。 + +### 2. 替换静态资源 location + +这个 location 已有自己的 add_header，所以必须在这里也设置 CORS 和 Vary，不能只依赖 server 层。保留原来的文件路径、缓存、HSTS 和字符集： + +```nginx +location ~* \.(?:jpg|jpeg|png|gif|webp|avif|svg|ico|bmp|swf|css|js|woff|woff2|ttf|eot)$ { + root /dcalc/image; + expires 7d; + add_header Cache-Control "public, max-age=604800"; + add_header Strict-Transport-Security "max-age=31536000"; + + add_header Access-Control-Allow-Origin $image_cors_origin always; + add_header Vary "Origin" always; + add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always; + add_header Access-Control-Allow-Headers "If-None-Match, If-Modified-Since" always; + add_header Access-Control-Max-Age "86400" always; + + if ($request_method = OPTIONS) { + return 204; + } + + try_files $uri =404; + charset utf-8,gbk; +} +``` + +`always` 使 CORS 头也适用于 304 和错误响应；原配置的 `error_page 404 /404.html` 内部跳转会由 `location /` 继承 server 层的 CORS 和 Vary 头。其余 location、证书和 HTTP 跳转配置保持不变。不要再保留之前的 `Access-Control-Allow-Origin "*"`，也不要重复添加此头。 + +这是对浏览器跨域脚本读取的限制，不是防盗链或访问鉴权；其他来源仍可能展示普通 img，或直接下载公开图片。当前导出无需 Cookie，不需要 Access-Control-Allow-Credentials。 + +### 3. 检查、重载及验证 + +在实际运行 Nginx / OpenResty 的环境中执行（容器部署则在对应容器内执行）： + +```sh +nginx -t +nginx -s reload + +# 应返回 Access-Control-Allow-Origin: https://dcalc.dnftools.com +# 并包含 Vary: Origin +curl -I -H 'Origin: https://dcalc.dnftools.com' \ + 'https://image.dnftools.com/characters/gunner_female/spitfire_female/profile.png' + +# 不应返回 Access-Control-Allow-Origin +curl -I -H 'Origin: https://dnftools.com.example.org' \ + 'https://image.dnftools.com/characters/gunner_female/spitfire_female/profile.png' +``` + +如果前面还有 CDN，需让其遵守 Vary / 按 Origin 区分缓存，或在边缘执行相同的来源校验，并刷新旧响应头缓存。最后回到技能设置页导出，检查 PNG 中包含图标、等级和觉醒绑定。 + +## 仓库自带的 Node 图片服务 + +`pnpm run dev:image` 使用 `scripts/image.js`，已采用同样的来源规则：匹配时返回请求 Origin，其他来源不返回 Access-Control-Allow-Origin；所有响应设置 Vary: Origin，保留 HEAD、304、错误响应和 OPTIONS 支持。 + +本地前端通过 Vite 的 `/image` 同源代理访问图片，不需要放行 localhost。部署使用 Node 服务时需要重启该进程；当前生产配置直接通过 Nginx 读取 `/dcalc/image`，所以生产生效点是上面的 Nginx 修改，无需重新构建前端。 + +参考：[CORS 来源与缓存](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Access-Control-Allow-Origin)、[Nginx 响应头继承](https://nginx.org/en/docs/http/ngx_http_headers_module.html)、[Nginx set / if](https://nginx.org/en/docs/http/ngx_http_rewrite_module.html)。 diff --git a/scripts/image.js b/scripts/image.js index 7b981db0..97e50103 100644 --- a/scripts/image.js +++ b/scripts/image.js @@ -12,6 +12,8 @@ const browserCacheSeconds = Number.parseInt( process.env.IMAGE_BROWSER_CACHE_SECONDS || '3600', 10, ); +// Match complete origins, not substrings such as dnftools.com.example.org. +const allowedImageOrigin = /^https?:\/\/([a-z0-9-]+\.)+dnftools\.com(:[0-9]+)?$/i; const mimeTypes = { '.avif': 'image/avif', @@ -123,8 +125,25 @@ function sendText(res, statusCode, message) { } const ImageServer = http.createServer(async (req, res) => { + // Exporting DOM to an image fetches icons across origins. Reflect only trusted + // origins, including on cached (304) and error responses. + res.setHeader('Vary', 'Origin'); + const origin = req.headers.origin; + if (origin && allowedImageOrigin.test(origin)) { + res.setHeader('Access-Control-Allow-Origin', origin); + } + + if (req.method === 'OPTIONS') { + res.setHeader('Access-Control-Allow-Methods', 'GET, HEAD, OPTIONS'); + res.setHeader('Access-Control-Allow-Headers', 'If-None-Match, If-Modified-Since'); + res.setHeader('Access-Control-Max-Age', '86400'); + res.writeHead(204); + res.end(); + return; + } + if (req.method !== 'GET' && req.method !== 'HEAD') { - res.setHeader('Allow', 'GET, HEAD'); + res.setHeader('Allow', 'GET, HEAD, OPTIONS'); sendText(res, 405, 'Method not allowed'); return; }