update
This commit is contained in:
+12
-3
@@ -32,9 +32,17 @@ uv run ruff check .
|
|||||||
## Gold Cache and Notifications
|
## Gold Cache and Notifications
|
||||||
|
|
||||||
- Application startup checks both realtime and official gold caches, then checks
|
- Application startup checks both realtime and official gold caches, then checks
|
||||||
them every minute. Data older than 30 minutes refreshes without HTTP traffic.
|
them at wall-clock minute boundaries. Each half-hour batch starts 5 minutes
|
||||||
|
early: 01:25 collects the 01:30 batch, 01:55 collects the 02:00 batch, without
|
||||||
|
HTTP traffic. All regions in a refresh share the batch timestamp selected at
|
||||||
|
the start, even if the work crosses a half-hour boundary. Existing public
|
||||||
|
`updataTime` types remain number for realtime and string for official.
|
||||||
Redis locks prevent duplicate work across workers. Realtime region pages stay
|
Redis locks prevent duplicate work across workers. Realtime region pages stay
|
||||||
8 seconds apart. Failures retain the last successful data for up to 24 hours.
|
8 seconds apart. Failures retain the last successful data for up to 24 hours.
|
||||||
|
- Realtime uses the arithmetic mean of displayed unit prices from rows 4–10
|
||||||
|
(`i.item1`, slice `[3:10]`), in 元/万金币, rounded to 2 decimals. This replaces
|
||||||
|
the earlier 万金币/元 average and uses a new Redis namespace so old units are
|
||||||
|
never returned as new prices. No BeautifulSoup or browser dependency is added.
|
||||||
- Official authentication uses `HELPER_TOKEN` and `HELPER_ID`. Authentication
|
- Official authentication uses `HELPER_TOKEN` and `HELPER_ID`. Authentication
|
||||||
failures (`returnCode=-30003`, explicit token-expiry messages or HTTP 401) can
|
failures (`returnCode=-30003`, explicit token-expiry messages or HTTP 401) can
|
||||||
notify via Bark. Network errors do not trigger token-expiry notifications.
|
notify via Bark. Network errors do not trigger token-expiry notifications.
|
||||||
@@ -46,8 +54,9 @@ uv run ruff check .
|
|||||||
included in notification text or logged push errors.
|
included in notification text or logged push errors.
|
||||||
- Official gold responses also contain `max` and `min`: the largest/smallest
|
- Official gold responses also contain `max` and `min`: the largest/smallest
|
||||||
values of `10000 / price` across both rank arrays, in 万金币 per 100点券,
|
values of `10000 / price` across both rank arrays, in 万金币 per 100点券,
|
||||||
rounded to 4 decimals. Nonpositive/invalid prices are excluded; absent valid
|
rounded to 2 decimals. Nonpositive/invalid prices are excluded; absent valid
|
||||||
quotes yield `null`. Existing rank arrays retain their 点券 per 100万金币 unit.
|
quotes yield `null`. Rank arrays retain their 点券 per 100万金币 unit and are
|
||||||
|
also rounded to 2 decimals. JSON numbers do not force trailing zeroes.
|
||||||
|
|
||||||
## Compatibility Rules
|
## Compatibility Rules
|
||||||
|
|
||||||
|
|||||||
@@ -41,9 +41,9 @@ def get_date_str(timestamp: int) -> str:
|
|||||||
|
|
||||||
@router.get('/gold/realtime/', operation_id='realtimeGold', response_model=Response[list[RealTimeGold]])
|
@router.get('/gold/realtime/', operation_id='realtimeGold', response_model=Response[list[RealTimeGold]])
|
||||||
async def get_real_time_gold(redis: Annotated[AsyncRedis, Depends(thirdpart_gold.get_redis)], background_tasks: BackgroundTasks) -> Response[list[RealTimeGold]]:
|
async def get_real_time_gold(redis: Annotated[AsyncRedis, Depends(thirdpart_gold.get_redis)], background_tasks: BackgroundTasks) -> Response[list[RealTimeGold]]:
|
||||||
"""获取网站金币比例,数据为各跨区首次报价前200条的平均万金币/元。
|
"""获取网站金币单价,取页面第4~10条单价均值,元/万金币,两位小数。
|
||||||
|
|
||||||
updataTime为数字秒级时间戳,code为跨区标识。与thirdpart接口共用Redis缓存和
|
updataTime为数据所属半小时刻度的数字秒级时间戳,:25/:55提前采集对应:30/:00数据。与thirdpart接口共用Redis缓存和
|
||||||
后台刷新任务:30分钟自动更新,也支持按需补齐,逐区间隔8秒;初始化未齐时响应体code=202。
|
后台刷新任务:30分钟自动更新,也支持按需补齐,逐区间隔8秒;初始化未齐时响应体code=202。
|
||||||
"""
|
"""
|
||||||
return await get_thirdpart_gold(redis, background_tasks)
|
return await get_thirdpart_gold(redis, background_tasks)
|
||||||
@@ -145,11 +145,11 @@ async def get_official_gold(redis: Annotated[AsyncRedis, Depends(thirdpart_gold.
|
|||||||
|
|
||||||
@router.get('/gold/thirdpart/', operation_id='thirdpartGold', response_model=Response[list[RealTimeGold]])
|
@router.get('/gold/thirdpart/', operation_id='thirdpartGold', response_model=Response[list[RealTimeGold]])
|
||||||
async def get_thirdpart_gold(redis: Annotated[AsyncRedis, Depends(thirdpart_gold.get_redis)], background_tasks: BackgroundTasks) -> Response[list[RealTimeGold]]:
|
async def get_thirdpart_gold(redis: Annotated[AsyncRedis, Depends(thirdpart_gold.get_redis)], background_tasks: BackgroundTasks) -> Response[list[RealTimeGold]]:
|
||||||
"""获取跨1~跨8(含跨3A/3B)的第三方首批报价均值,单位:万金币/元。
|
"""获取跨1~跨8(含跨3A/3B)的第三方单价均值,单位:元/万金币。
|
||||||
|
|
||||||
不限制最低总价;各平台首次返回的有效报价汇总、去重、按单价取前200条,
|
不额外限制最低总价;汇总渠道报价并按公开页面展示规则排序、去重,
|
||||||
对每条报价的1/Price取算术平均,不按库存或份数加权。保留上游原始价格,
|
对i.item1的第4~10条(Python切片[3:10])显示单价取算术平均,
|
||||||
不额外计算交易手续费。该接口无上游分页参数,前200条是本接口的样本口径。
|
四舍五入至两位小数,不按库存或份数加权。
|
||||||
|
|
||||||
每30分钟自动后台刷新,跨区页面最短间隔8秒(每分钟不超过8次),Redis锁避免重复刷新。
|
每30分钟自动后台刷新,跨区页面最短间隔8秒(每分钟不超过8次),Redis锁避免重复刷新。
|
||||||
初始化跨区未齐时响应体code=202并注明进度;9区缓存齐全才返回code=200。
|
初始化跨区未齐时响应体code=202并注明进度;9区缓存齐全才返回code=200。
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field, field_validator
|
||||||
|
|
||||||
|
from api.utils.gold_time import half_hour_start
|
||||||
|
|
||||||
|
|
||||||
GOLD_REGION_CODES = {
|
GOLD_REGION_CODES = {
|
||||||
@@ -8,20 +10,29 @@ GOLD_REGION_CODES = {
|
|||||||
|
|
||||||
|
|
||||||
class RealTimeGold(BaseModel):
|
class RealTimeGold(BaseModel):
|
||||||
updataTime: int = Field(..., ge=0, description='更新时间,Unix秒级时间戳')
|
updataTime: int = Field(..., ge=0, description='数据所属半小时刻度的Unix秒级时间戳,提前5分钟开始采集')
|
||||||
"""更新时间"""
|
"""更新时间"""
|
||||||
region: str = Field(..., description='跨区')
|
region: str = Field(..., description='跨区')
|
||||||
"""跨区"""
|
"""跨区"""
|
||||||
code: str = Field(..., description='跨区code:k1、k2、k3a、k3b、k4、k5、k6、k7、k8')
|
code: str = Field(..., description='跨区code:k1、k2、k3a、k3b、k4、k5、k6、k7、k8')
|
||||||
goldPrice: float = Field(..., gt=0, allow_inf_nan=False, description='金币比例,1元对应的万金币数量')
|
goldPrice: float = Field(..., ge=0, allow_inf_nan=False, description='页面第4~10条单价均值,元/万金币,四舍五入至两位小数')
|
||||||
"""金价"""
|
"""金价"""
|
||||||
|
|
||||||
|
@field_validator('updataTime')
|
||||||
|
@classmethod
|
||||||
|
def align_update_time(cls, value):
|
||||||
|
return half_hour_start(value)
|
||||||
|
|
||||||
|
@field_validator('goldPrice')
|
||||||
|
@classmethod
|
||||||
|
def round_gold_price(cls, value):
|
||||||
|
return round(value, 2)
|
||||||
|
|
||||||
|
|
||||||
class ThirdPartGoldCache(RealTimeGold):
|
class ThirdPartGoldCache(RealTimeGold):
|
||||||
"""内部缓存数据,不作为接口返回模型。"""
|
"""内部缓存数据,不作为接口返回模型。"""
|
||||||
|
|
||||||
goldPrice: float = Field(..., gt=0, allow_inf_nan=False, description='首页有效报价换算为万金币/元后的算术平均值')
|
sampleCount: int = Field(..., ge=1, le=7, description='第4~10条中参与均值计算的报价条数')
|
||||||
sampleCount: int = Field(..., ge=1, le=200, description='参与均值计算的报价条数')
|
|
||||||
stale: bool = Field(..., description='数据是否已超过30分钟刷新周期')
|
stale: bool = Field(..., description='数据是否已超过30分钟刷新周期')
|
||||||
|
|
||||||
|
|
||||||
@@ -41,7 +52,7 @@ class HistoryGold(BaseModel):
|
|||||||
"""echarts数据"""
|
"""echarts数据"""
|
||||||
|
|
||||||
class OfficialGold(BaseModel):
|
class OfficialGold(BaseModel):
|
||||||
updataTime: str = Field(..., pattern=r'^\d+$', description='更新时间,Unix秒级时间戳字符串')
|
updataTime: str = Field(..., pattern=r'^\d+$', description='数据所属半小时刻度的Unix秒级时间戳字符串,提前5分钟开始采集')
|
||||||
"""更新时间"""
|
"""更新时间"""
|
||||||
region: str = Field(..., description='跨区')
|
region: str = Field(..., description='跨区')
|
||||||
"""跨区"""
|
"""跨区"""
|
||||||
@@ -52,3 +63,18 @@ class OfficialGold(BaseModel):
|
|||||||
"""金币寄售最高成交价排名前5的价格"""
|
"""金币寄售最高成交价排名前5的价格"""
|
||||||
max: float | None = Field(..., description='100点券可兑换的最大万金币数量,无有效报价时为null')
|
max: float | None = Field(..., description='100点券可兑换的最大万金币数量,无有效报价时为null')
|
||||||
min: float | None = Field(..., description='100点券可兑换的最小万金币数量,无有效报价时为null')
|
min: float | None = Field(..., description='100点券可兑换的最小万金币数量,无有效报价时为null')
|
||||||
|
|
||||||
|
@field_validator('updataTime')
|
||||||
|
@classmethod
|
||||||
|
def align_update_time(cls, value):
|
||||||
|
return str(half_hour_start(int(value)))
|
||||||
|
|
||||||
|
@field_validator('goldDealRank', 'goldTopDealRank')
|
||||||
|
@classmethod
|
||||||
|
def round_ranks(cls, values):
|
||||||
|
return [round(value, 2) for value in values]
|
||||||
|
|
||||||
|
@field_validator('max', 'min')
|
||||||
|
@classmethod
|
||||||
|
def round_ratios(cls, value):
|
||||||
|
return round(value, 2) if value is not None else None
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
|
import time
|
||||||
|
|
||||||
from api.utils import official_gold, thirdpart_gold
|
from api.utils import official_gold, thirdpart_gold
|
||||||
|
|
||||||
@@ -27,7 +28,8 @@ async def _periodic(job):
|
|||||||
await job()
|
await job()
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception('periodic gold refresh check failed job=%s', job.__name__)
|
logger.exception('periodic gold refresh check failed job=%s', job.__name__)
|
||||||
await asyncio.sleep(CHECK_INTERVAL_SECONDS)
|
# 对齐墙钟分钟边界,:25/:55检测到下一半小时数据周期并提前采集。
|
||||||
|
await asyncio.sleep(CHECK_INTERVAL_SECONDS - time.time() % CHECK_INTERVAL_SECONDS)
|
||||||
|
|
||||||
|
|
||||||
async def start_periodic_refresh():
|
async def start_periodic_refresh():
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
"""每个半小时刻度提前5分钟采集,返回值标记对应的半小时刻度。"""
|
||||||
|
|
||||||
|
import time
|
||||||
|
|
||||||
|
HALF_HOUR_SECONDS = 30 * 60
|
||||||
|
UPDATE_LEAD_SECONDS = 5 * 60
|
||||||
|
|
||||||
|
|
||||||
|
def half_hour_start(timestamp: float | None = None) -> int:
|
||||||
|
if timestamp is None:
|
||||||
|
timestamp = time.time()
|
||||||
|
return int(timestamp) // HALF_HOUR_SECONDS * HALF_HOUR_SECONDS
|
||||||
|
|
||||||
|
|
||||||
|
def refresh_slot(timestamp: float | None = None) -> int:
|
||||||
|
if timestamp is None:
|
||||||
|
timestamp = time.time()
|
||||||
|
return half_hour_start(timestamp + UPDATE_LEAD_SECONDS)
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
"""官方寄售缓存、认证失败识别及可选BARK通知。"""
|
"""官方寄售缓存、认证失败识别及可选BARK通知。"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from datetime import datetime, timezone
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
@@ -14,6 +13,7 @@ import aiohttp
|
|||||||
from api.core.Gzip import gzip_unzip, gzip_zip
|
from api.core.Gzip import gzip_unzip, gzip_zip
|
||||||
from api.types.colg import GOLD_REGION_CODES, OfficialGold
|
from api.types.colg import GOLD_REGION_CODES, OfficialGold
|
||||||
from api.utils.thirdpart_gold import RELEASE_LOCK
|
from api.utils.thirdpart_gold import RELEASE_LOCK
|
||||||
|
from api.utils.gold_time import UPDATE_LEAD_SECONDS, half_hour_start, refresh_slot
|
||||||
from config.main import config
|
from config.main import config
|
||||||
|
|
||||||
logger = logging.getLogger('dcalc.official_gold')
|
logger = logging.getLogger('dcalc.official_gold')
|
||||||
@@ -43,14 +43,14 @@ def normalize_row(item: dict) -> dict:
|
|||||||
except (TypeError, ValueError, OverflowError):
|
except (TypeError, ValueError, OverflowError):
|
||||||
continue
|
continue
|
||||||
if math.isfinite(price):
|
if math.isfinite(price):
|
||||||
prices.append(price)
|
prices.append(round(price, 2))
|
||||||
ranks[name] = prices
|
ranks[name] = prices
|
||||||
ratios = [10000 / price for prices in ranks.values() for price in prices
|
ratios = [10000 / price for prices in ranks.values() for price in prices
|
||||||
if price > 0 and math.isfinite(10000 / price)]
|
if price > 0 and math.isfinite(10000 / price)]
|
||||||
return OfficialGold.model_validate({
|
return OfficialGold.model_validate({
|
||||||
**item, **ranks,
|
**item, **ranks, 'updataTime': str(half_hour_start(int(item['updataTime']))),
|
||||||
'max': round(max(ratios), 4) if ratios else None,
|
'max': round(max(ratios), 2) if ratios else None,
|
||||||
'min': round(min(ratios), 4) if ratios else None,
|
'min': round(min(ratios), 2) if ratios else None,
|
||||||
}).model_dump()
|
}).model_dump()
|
||||||
|
|
||||||
|
|
||||||
@@ -64,7 +64,10 @@ def is_auth_failure(payload: dict) -> bool:
|
|||||||
))
|
))
|
||||||
|
|
||||||
|
|
||||||
async def fetch_gold() -> list[dict]:
|
async def fetch_gold(*, started_at: float | None = None) -> list[dict]:
|
||||||
|
if started_at is None:
|
||||||
|
started_at = time.time()
|
||||||
|
update_time = str(refresh_slot(started_at))
|
||||||
timeout = aiohttp.ClientTimeout(total=10)
|
timeout = aiohttp.ClientTimeout(total=10)
|
||||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||||
async def fetch_region(region):
|
async def fetch_region(region):
|
||||||
@@ -81,7 +84,7 @@ async def fetch_gold() -> list[dict]:
|
|||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
data = payload['data']
|
data = payload['data']
|
||||||
return normalize_row({
|
return normalize_row({
|
||||||
'updataTime': str(int(datetime.now(timezone.utc).timestamp())),
|
'updataTime': update_time,
|
||||||
'region': f'跨{region}', 'code': GOLD_REGION_CODES[f'跨{region}'],
|
'region': f'跨{region}', 'code': GOLD_REGION_CODES[f'跨{region}'],
|
||||||
'goldDealRank': [float(item['price']) for item in data['godDealRank'][:5]],
|
'goldDealRank': [float(item['price']) for item in data['godDealRank'][:5]],
|
||||||
'goldTopDealRank': [float(item['price']) for item in data['godTopDealRank'][:5]],
|
'goldTopDealRank': [float(item['price']) for item in data['godTopDealRank'][:5]],
|
||||||
@@ -150,7 +153,7 @@ async def read_cache(redis) -> list[dict]:
|
|||||||
for item in json.loads(gzip_unzip(value))]
|
for item in json.loads(gzip_unzip(value))]
|
||||||
if [row['region'] for row in rows] != [f'跨{region}' for region in REGIONS]:
|
if [row['region'] for row in rows] != [f'跨{region}' for region in REGIONS]:
|
||||||
return []
|
return []
|
||||||
if any(not 0 <= time.time() - int(row['updataTime']) < RETENTION_SECONDS for row in rows):
|
if any(not -UPDATE_LEAD_SECONDS <= time.time() - int(row['updataTime']) < RETENTION_SECONDS for row in rows):
|
||||||
return []
|
return []
|
||||||
return rows
|
return rows
|
||||||
except (ValueError, TypeError, KeyError, OSError):
|
except (ValueError, TypeError, KeyError, OSError):
|
||||||
@@ -160,7 +163,7 @@ async def read_cache(redis) -> list[dict]:
|
|||||||
|
|
||||||
def is_fresh(rows: list[dict]) -> bool:
|
def is_fresh(rows: list[dict]) -> bool:
|
||||||
return (len(rows) == len(REGIONS)
|
return (len(rows) == len(REGIONS)
|
||||||
and all(0 <= time.time() - int(row['updataTime']) < FRESH_SECONDS for row in rows))
|
and all(int(row['updataTime']) == refresh_slot() for row in rows))
|
||||||
|
|
||||||
|
|
||||||
async def refresh_cache(redis) -> bool:
|
async def refresh_cache(redis) -> bool:
|
||||||
@@ -173,7 +176,7 @@ async def refresh_cache(redis) -> bool:
|
|||||||
if is_fresh(await read_cache(redis)) or await redis.get(COOLDOWN_KEY):
|
if is_fresh(await read_cache(redis)) or await redis.get(COOLDOWN_KEY):
|
||||||
return False
|
return False
|
||||||
try:
|
try:
|
||||||
data = await fetch_gold()
|
data = await fetch_gold(started_at=time.time())
|
||||||
except TokenExpired:
|
except TokenExpired:
|
||||||
logger.warning('official gold authentication failed; check HELPER_TOKEN and HELPER_ID')
|
logger.warning('official gold authentication failed; check HELPER_TOKEN and HELPER_ID')
|
||||||
await notify_token_expired(redis)
|
await notify_token_expired(redis)
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ import math
|
|||||||
import re
|
import re
|
||||||
from statistics import fmean
|
from statistics import fmean
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timedelta, timezone
|
|
||||||
from urllib.parse import urljoin, urlsplit
|
from urllib.parse import urljoin, urlsplit
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
@@ -19,6 +18,7 @@ from redis.asyncio.retry import Retry
|
|||||||
from redis.backoff import NoBackoff
|
from redis.backoff import NoBackoff
|
||||||
|
|
||||||
from api.types.colg import GOLD_REGION_CODES, ThirdPartGoldCache
|
from api.types.colg import GOLD_REGION_CODES, ThirdPartGoldCache
|
||||||
|
from api.utils.gold_time import refresh_slot
|
||||||
from config.main import config
|
from config.main import config
|
||||||
|
|
||||||
logger = logging.getLogger('dcalc.thirdpart_gold')
|
logger = logging.getLogger('dcalc.thirdpart_gold')
|
||||||
@@ -36,6 +36,7 @@ HEADERS = {
|
|||||||
# 网站公开 DNF 页面已停用/移除的渠道;不请求 VIP 专属渠道。
|
# 网站公开 DNF 页面已停用/移除的渠道;不请求 VIP 专属渠道。
|
||||||
DISABLED_PLATFORMS = {1, 12, 21, 50, 51, 52, 53}
|
DISABLED_PLATFORMS = {1, 12, 21, 50, 51, 52, 53}
|
||||||
PAGE_SIZE = 200
|
PAGE_SIZE = 200
|
||||||
|
SAMPLE_SLICE = slice(3, 10)
|
||||||
FRESH_SECONDS = 30 * 60
|
FRESH_SECONDS = 30 * 60
|
||||||
RETENTION_SECONDS = 24 * 60 * 60
|
RETENTION_SECONDS = 24 * 60 * 60
|
||||||
# 按每分钟最多8次页面访问处理,理论间隔7.5秒,取8秒留出边界余量。
|
# 按每分钟最多8次页面访问处理,理论间隔7.5秒,取8秒留出边界余量。
|
||||||
@@ -44,7 +45,8 @@ REGION_INTERVAL = 8
|
|||||||
LOCK_SECONDS = 15 * 60
|
LOCK_SECONDS = 15 * 60
|
||||||
ERROR_COOLDOWN_SECONDS = 5 * 60
|
ERROR_COOLDOWN_SECONDS = 5 * 60
|
||||||
REDIS_TIMEOUT_SECONDS = 1
|
REDIS_TIMEOUT_SECONDS = 1
|
||||||
CACHE_PREFIX = 'openapi:colg:thirdpart_gold:v1'
|
# 统计范围和单位已改变,旧缓存的万金币/元均值不能作为新的元/万金币单价返回。
|
||||||
|
CACHE_PREFIX = 'openapi:colg:thirdpart_gold:v2'
|
||||||
LOCK_KEY = f'{CACHE_PREFIX}:lock'
|
LOCK_KEY = f'{CACHE_PREFIX}:lock'
|
||||||
COOLDOWN_KEY = f'{CACHE_PREFIX}:cooldown'
|
COOLDOWN_KEY = f'{CACHE_PREFIX}:cooldown'
|
||||||
RELEASE_LOCK = """
|
RELEASE_LOCK = """
|
||||||
@@ -126,7 +128,7 @@ def make_payload(channel: dict, seed: dict, user: dict) -> dict:
|
|||||||
|
|
||||||
|
|
||||||
def first_page_average(offers: list[dict]) -> tuple[float, int]:
|
def first_page_average(offers: list[dict]) -> tuple[float, int]:
|
||||||
"""等权平均每条报价的 1/Price(万金币/元),不是 1/平均单价。"""
|
"""匹配公开页面 i.item1 的第4~10条,平均显示单价(元/万金币)。"""
|
||||||
valid = []
|
valid = []
|
||||||
for offer in offers:
|
for offer in offers:
|
||||||
try:
|
try:
|
||||||
@@ -135,9 +137,12 @@ def first_page_average(offers: list[dict]) -> tuple[float, int]:
|
|||||||
number = float(offer['Number'])
|
number = float(offer['Number'])
|
||||||
except (KeyError, TypeError, ValueError, OverflowError):
|
except (KeyError, TypeError, ValueError, OverflowError):
|
||||||
continue
|
continue
|
||||||
if not all(math.isfinite(value) and value > 0 for value in (price, money, number)):
|
if not all(math.isfinite(value) for value in (price, money, number)) or price < 0:
|
||||||
continue
|
continue
|
||||||
if not math.isfinite(1 / price):
|
# 页面默认展示条件,不额外设置price1/price2价格过滤。
|
||||||
|
if money != -1 and (money * number < 5 or price == 0):
|
||||||
|
continue
|
||||||
|
if money >= 1_000_000 or (offer.get('Seller') or '').startswith('智莹科技'):
|
||||||
continue
|
continue
|
||||||
valid.append({**offer, 'Price': price})
|
valid.append({**offer, 'Price': price})
|
||||||
valid.sort(key=lambda item: (item['Price'], item['PfId'], item.get('BuyUrl') or ''))
|
valid.sort(key=lambda item: (item['Price'], item['PfId'], item.get('BuyUrl') or ''))
|
||||||
@@ -152,9 +157,12 @@ def first_page_average(offers: list[dict]) -> tuple[float, int]:
|
|||||||
page.append(offer)
|
page.append(offer)
|
||||||
if len(page) == PAGE_SIZE:
|
if len(page) == PAGE_SIZE:
|
||||||
break
|
break
|
||||||
if not page:
|
selected = page[SAMPLE_SLICE]
|
||||||
|
if not selected:
|
||||||
raise ValueError('YXDR 未返回有效报价')
|
raise ValueError('YXDR 未返回有效报价')
|
||||||
return round(fmean(1 / item['Price'] for item in page), 4), len(page)
|
# 页面 Math.floorDigits(price, 5) 先toFixed(6)再删掉末位。
|
||||||
|
prices = [float(f"{item['Price']:.6f}"[:-1]) for item in selected]
|
||||||
|
return round(fmean(prices), 2), len(prices)
|
||||||
|
|
||||||
|
|
||||||
async def _read_response(resp) -> str:
|
async def _read_response(resp) -> str:
|
||||||
@@ -165,7 +173,9 @@ async def _read_response(resp) -> str:
|
|||||||
return text
|
return text
|
||||||
|
|
||||||
|
|
||||||
async def fetch_region(session: aiohttp.ClientSession, region: str) -> dict:
|
async def fetch_region(session: aiohttp.ClientSession, region: str, *, started_at: float | None = None) -> dict:
|
||||||
|
if started_at is None:
|
||||||
|
started_at = time.time()
|
||||||
page_url = f'{BASE_URL}/bijiaqi/dnf/youxibi/{region}'
|
page_url = f'{BASE_URL}/bijiaqi/dnf/youxibi/{region}'
|
||||||
async with session.get(page_url) as resp:
|
async with session.get(page_url) as resp:
|
||||||
page = await _read_response(resp)
|
page = await _read_response(resp)
|
||||||
@@ -210,10 +220,9 @@ async def fetch_region(session: aiohttp.ClientSession, region: str) -> dict:
|
|||||||
if isinstance(result, BaseException):
|
if isinstance(result, BaseException):
|
||||||
raise result
|
raise result
|
||||||
price, count = first_page_average([offer for result in results for offer in result])
|
price, count = first_page_average([offer for result in results for offer in result])
|
||||||
now = datetime.now(timezone(timedelta(hours=8)))
|
|
||||||
return {'region': REGIONS[region], 'code': GOLD_REGION_CODES[REGIONS[region]],
|
return {'region': REGIONS[region], 'code': GOLD_REGION_CODES[REGIONS[region]],
|
||||||
'goldPrice': price, 'sampleCount': count,
|
'goldPrice': price, 'sampleCount': count,
|
||||||
'updataTime': int(now.timestamp()), 'fetchedAt': now.timestamp()}
|
'updataTime': refresh_slot(started_at), 'fetchedAt': started_at}
|
||||||
|
|
||||||
|
|
||||||
async def read_cache(redis) -> list[dict]:
|
async def read_cache(redis) -> list[dict]:
|
||||||
@@ -229,8 +238,9 @@ async def read_cache(redis) -> list[dict]:
|
|||||||
continue
|
continue
|
||||||
# 兼容此前缓存的格式化日期及缺少code的记录,无须重新抓取所有跨区。
|
# 兼容此前缓存的格式化日期及缺少code的记录,无须重新抓取所有跨区。
|
||||||
result.append(ThirdPartGoldCache.model_validate({
|
result.append(ThirdPartGoldCache.model_validate({
|
||||||
**item, 'updataTime': int(item['fetchedAt']),
|
**item, 'updataTime': refresh_slot(item['fetchedAt']),
|
||||||
'code': GOLD_REGION_CODES[REGIONS[region]], 'stale': age >= FRESH_SECONDS,
|
'code': GOLD_REGION_CODES[REGIONS[region]],
|
||||||
|
'stale': refresh_slot(item['fetchedAt']) < refresh_slot(),
|
||||||
}).model_dump())
|
}).model_dump())
|
||||||
except (KeyError, TypeError, ValueError):
|
except (KeyError, TypeError, ValueError):
|
||||||
logger.warning('invalid thirdpart gold cache region=%s', region)
|
logger.warning('invalid thirdpart gold cache region=%s', region)
|
||||||
@@ -270,6 +280,7 @@ async def refresh_cache(token: str):
|
|||||||
fresh_regions = {item['region'] for item in cached if not item['stale']}
|
fresh_regions = {item['region'] for item in cached if not item['stale']}
|
||||||
pending = [region for region, name in REGIONS.items() if name not in fresh_regions]
|
pending = [region for region, name in REGIONS.items() if name not in fresh_regions]
|
||||||
successful, last_start = 0, None
|
successful, last_start = 0, None
|
||||||
|
started_at = time.time()
|
||||||
timeout = aiohttp.ClientTimeout(total=15)
|
timeout = aiohttp.ClientTimeout(total=15)
|
||||||
async with aiohttp.ClientSession(headers=HEADERS, timeout=timeout,
|
async with aiohttp.ClientSession(headers=HEADERS, timeout=timeout,
|
||||||
connector=aiohttp.TCPConnector(limit=5)) as session:
|
connector=aiohttp.TCPConnector(limit=5)) as session:
|
||||||
@@ -281,7 +292,7 @@ async def refresh_cache(token: str):
|
|||||||
return
|
return
|
||||||
last_start = time.monotonic()
|
last_start = time.monotonic()
|
||||||
try:
|
try:
|
||||||
item = await fetch_region(session, region)
|
item = await fetch_region(session, region, started_at=started_at)
|
||||||
if await redis.get(LOCK_KEY) not in (token, token.encode()):
|
if await redis.get(LOCK_KEY) not in (token, token.encode()):
|
||||||
return
|
return
|
||||||
await redis.set(f'{CACHE_PREFIX}:{region}', json.dumps(item),
|
await redis.set(f'{CACHE_PREFIX}:{region}', json.dumps(item),
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import pytest
|
|||||||
|
|
||||||
from api.utils import official_gold as gold
|
from api.utils import official_gold as gold
|
||||||
from api.core.Gzip import gzip_zip
|
from api.core.Gzip import gzip_zip
|
||||||
|
from api.utils.gold_time import refresh_slot
|
||||||
|
|
||||||
|
|
||||||
class FakeResponse:
|
class FakeResponse:
|
||||||
@@ -67,14 +68,37 @@ def test_fetch_official_gold_uses_async_http_with_isolated_forms(monkeypatch):
|
|||||||
assert [item['region'] for item in result] == [f'跨{region}' for region in gold.REGIONS]
|
assert [item['region'] for item in result] == [f'跨{region}' for region in gold.REGIONS]
|
||||||
assert [item['code'] for item in result] == ['k1', 'k2', 'k3a', 'k3b', 'k4', 'k5', 'k6', 'k7']
|
assert [item['code'] for item in result] == ['k1', 'k2', 'k3a', 'k3b', 'k4', 'k5', 'k6', 'k7']
|
||||||
assert all(isinstance(item['updataTime'], str) and item['updataTime'].isdigit() for item in result)
|
assert all(isinstance(item['updataTime'], str) and item['updataTime'].isdigit() for item in result)
|
||||||
assert abs(int(result[0]['updataTime']) - time.time()) < 2
|
assert int(result[0]['updataTime']) == refresh_slot()
|
||||||
assert result[0]['goldDealRank'] == [100.5, 101, 102, 103, 104]
|
assert result[0]['goldDealRank'] == [100.5, 101, 102, 103, 104]
|
||||||
assert result[0]['goldTopDealRank'] == [200.25, 201, 202, 203, 204]
|
assert result[0]['goldTopDealRank'] == [200.25, 201, 202, 203, 204]
|
||||||
assert result[0]['max'] == round(10000 / 100.5, 4)
|
assert result[0]['max'] == round(10000 / 100.5, 2)
|
||||||
assert result[0]['min'] == round(10000 / 204, 4)
|
assert result[0]['min'] == round(10000 / 204, 2)
|
||||||
assert sessions[0].kwargs['timeout'].total == 10
|
assert sessions[0].kwargs['timeout'].total == 10
|
||||||
|
|
||||||
|
|
||||||
|
def test_official_batch_timestamp_comes_from_start_not_fetch_completion(monkeypatch):
|
||||||
|
started_at = 1793766300 # 与真实结束时间不同的固定开始时间。
|
||||||
|
monkeypatch.setattr(gold.aiohttp, 'ClientSession', lambda **kwargs: FakeSession([], **kwargs))
|
||||||
|
rows = asyncio.run(gold.fetch_gold(started_at=started_at))
|
||||||
|
assert {row['updataTime'] for row in rows} == {str(refresh_slot(started_at))}
|
||||||
|
|
||||||
|
|
||||||
|
def test_preloaded_next_slot_cache_is_servable_before_the_half_hour(monkeypatch):
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
china = timezone(timedelta(hours=8))
|
||||||
|
now = datetime(2026, 11, 4, 1, 25, tzinfo=china).timestamp()
|
||||||
|
monkeypatch.setattr(gold.time, 'time', lambda: now)
|
||||||
|
rows = cached_rows()
|
||||||
|
redis = FakeRedis()
|
||||||
|
redis.values[gold.CACHE_KEY] = gzip_zip(json.dumps(rows))
|
||||||
|
assert gold.is_fresh(asyncio.run(gold.read_cache(redis)))
|
||||||
|
target = int(rows[0]['updataTime'])
|
||||||
|
assert target > now
|
||||||
|
assert target - now == 300
|
||||||
|
monkeypatch.setattr(gold.time, 'time', lambda: now + 1800)
|
||||||
|
assert not gold.is_fresh(rows)
|
||||||
|
|
||||||
|
|
||||||
def test_official_route_uses_new_cache_contract_and_numeric_rank_arrays(monkeypatch, client):
|
def test_official_route_uses_new_cache_contract_and_numeric_rank_arrays(monkeypatch, client):
|
||||||
calls = []
|
calls = []
|
||||||
|
|
||||||
@@ -82,7 +106,7 @@ def test_official_route_uses_new_cache_contract_and_numeric_rank_arrays(monkeypa
|
|||||||
calls.append(redis)
|
calls.append(redis)
|
||||||
return [{'updataTime': '1793711716', 'region': '跨3B', 'code': 'k3b',
|
return [{'updataTime': '1793711716', 'region': '跨3B', 'code': 'k3b',
|
||||||
'goldDealRank': [100.5, 101], 'goldTopDealRank': [200.25, 201],
|
'goldDealRank': [100.5, 101], 'goldTopDealRank': [200.25, 201],
|
||||||
'max': round(10000 / 100.5, 4), 'min': round(10000 / 201, 4)}]
|
'max': round(10000 / 100.5, 2), 'min': round(10000 / 201, 2)}]
|
||||||
|
|
||||||
monkeypatch.setattr(gold, 'get_data', cached)
|
monkeypatch.setattr(gold, 'get_data', cached)
|
||||||
body = client.get('/api/open/gold/official/').json()
|
body = client.get('/api/open/gold/official/').json()
|
||||||
@@ -95,8 +119,8 @@ def test_official_route_uses_new_cache_contract_and_numeric_rank_arrays(monkeypa
|
|||||||
assert row['code'] == 'k3b'
|
assert row['code'] == 'k3b'
|
||||||
assert row['goldDealRank'] == [100.5, 101]
|
assert row['goldDealRank'] == [100.5, 101]
|
||||||
assert row['goldTopDealRank'] == [200.25, 201]
|
assert row['goldTopDealRank'] == [200.25, 201]
|
||||||
assert row['max'] == round(10000 / 100.5, 4)
|
assert row['max'] == round(10000 / 100.5, 2)
|
||||||
assert row['min'] == round(10000 / 201, 4)
|
assert row['min'] == round(10000 / 201, 2)
|
||||||
for rank in (row['goldDealRank'], row['goldTopDealRank']):
|
for rank in (row['goldDealRank'], row['goldTopDealRank']):
|
||||||
assert len(rank) <= 5
|
assert len(rank) <= 5
|
||||||
assert all(isinstance(value, (int, float)) for value in rank)
|
assert all(isinstance(value, (int, float)) for value in rank)
|
||||||
@@ -140,7 +164,7 @@ class FakeRedis:
|
|||||||
|
|
||||||
|
|
||||||
def cached_rows(age=0):
|
def cached_rows(age=0):
|
||||||
return [{'updataTime': str(int(time.time() - age)), 'region': f'跨{region}',
|
return [{'updataTime': str(refresh_slot(time.time() - age)), 'region': f'跨{region}',
|
||||||
'code': f'k{region.lower()}', 'goldDealRank': [100.5], 'goldTopDealRank': [200.25]}
|
'code': f'k{region.lower()}', 'goldDealRank': [100.5], 'goldTopDealRank': [200.25]}
|
||||||
for region in gold.REGIONS]
|
for region in gold.REGIONS]
|
||||||
|
|
||||||
@@ -169,8 +193,8 @@ def test_old_official_cache_gets_max_and_min_without_refetch():
|
|||||||
redis.values[gold.CACHE_KEY] = gzip_zip(json.dumps(cached_rows()))
|
redis.values[gold.CACHE_KEY] = gzip_zip(json.dumps(cached_rows()))
|
||||||
rows = asyncio.run(gold.read_cache(redis))
|
rows = asyncio.run(gold.read_cache(redis))
|
||||||
assert len(rows) == 8
|
assert len(rows) == 8
|
||||||
assert rows[0]['max'] == round(10000 / 100.5, 4)
|
assert rows[0]['max'] == round(10000 / 100.5, 2)
|
||||||
assert rows[0]['min'] == round(10000 / 200.25, 4)
|
assert rows[0]['min'] == round(10000 / 200.25, 2)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize('payload', [
|
@pytest.mark.parametrize('payload', [
|
||||||
@@ -206,7 +230,7 @@ def test_refresh_updates_cache_without_http_request_and_skips_when_fresh(monkeyp
|
|||||||
redis = FakeRedis()
|
redis = FakeRedis()
|
||||||
calls = []
|
calls = []
|
||||||
|
|
||||||
async def fetch():
|
async def fetch(*, started_at=None):
|
||||||
calls.append(1)
|
calls.append(1)
|
||||||
return cached_rows()
|
return cached_rows()
|
||||||
|
|
||||||
@@ -225,7 +249,7 @@ def test_failed_refresh_preserves_cache_and_notifies_once_per_fetch(monkeypatch)
|
|||||||
redis.values[gold.CACHE_KEY] = old
|
redis.values[gold.CACHE_KEY] = old
|
||||||
alerts = []
|
alerts = []
|
||||||
|
|
||||||
async def fetch():
|
async def fetch(*, started_at=None):
|
||||||
raise gold.TokenExpired()
|
raise gold.TokenExpired()
|
||||||
|
|
||||||
async def notify(redis):
|
async def notify(redis):
|
||||||
@@ -247,7 +271,7 @@ def test_network_failure_preserves_cache_without_token_alert(monkeypatch):
|
|||||||
old = gzip_zip(json.dumps(cached_rows(age=1900)))
|
old = gzip_zip(json.dumps(cached_rows(age=1900)))
|
||||||
redis.values[gold.CACHE_KEY] = old
|
redis.values[gold.CACHE_KEY] = old
|
||||||
|
|
||||||
async def fetch():
|
async def fetch(*, started_at=None):
|
||||||
raise TimeoutError('upstream unavailable')
|
raise TimeoutError('upstream unavailable')
|
||||||
|
|
||||||
async def notify(redis):
|
async def notify(redis):
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from api.types.colg import OfficialGold, RealTimeGold
|
||||||
|
from api.utils.gold_time import half_hour_start, refresh_slot
|
||||||
|
|
||||||
|
CHINA = timezone(timedelta(hours=8))
|
||||||
|
|
||||||
|
|
||||||
|
def stamp(hour, minute, second=0):
|
||||||
|
return int(datetime(2026, 11, 4, hour, minute, second, tzinfo=CHINA).timestamp())
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(('hour', 'minute', 'second', 'target_hour', 'target_minute'), [
|
||||||
|
(1, 24, 59, 1, 0), (1, 25, 0, 1, 30), (1, 29, 59, 1, 30),
|
||||||
|
(1, 30, 0, 1, 30), (1, 54, 59, 1, 30), (1, 55, 0, 2, 0), (2, 0, 0, 2, 0),
|
||||||
|
(23, 55, 0, 0, 0),
|
||||||
|
])
|
||||||
|
def test_refresh_slot_advances_five_minutes_before_boundary(hour, minute, second, target_hour, target_minute):
|
||||||
|
expected = stamp(target_hour, target_minute)
|
||||||
|
if hour == 23:
|
||||||
|
expected += 24 * 60 * 60
|
||||||
|
assert refresh_slot(stamp(hour, minute, second)) == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_response_types_round_price_values_and_keep_timestamp_types():
|
||||||
|
realtime = RealTimeGold(updataTime=stamp(1, 30), region='跨1', code='k1', goldPrice=.016789)
|
||||||
|
assert realtime.goldPrice == .02
|
||||||
|
assert realtime.updataTime == stamp(1, 30)
|
||||||
|
assert isinstance(realtime.updataTime, int)
|
||||||
|
official = OfficialGold(updataTime=str(stamp(1, 30)), region='跨1', code='k1',
|
||||||
|
goldDealRank=[125.556], goldTopDealRank=[250.123], max=79.64444, min=39.9876)
|
||||||
|
assert official.goldDealRank == [125.56]
|
||||||
|
assert official.goldTopDealRank == [250.12]
|
||||||
|
assert official.max == 79.64 and official.min == 39.99
|
||||||
|
assert official.updataTime == str(stamp(1, 30))
|
||||||
|
assert half_hour_start(stamp(1, 29, 59)) == stamp(1, 0)
|
||||||
@@ -9,16 +9,17 @@ from fastapi import BackgroundTasks
|
|||||||
from api.routes import colg
|
from api.routes import colg
|
||||||
from api.types.colg import GOLD_REGION_CODES
|
from api.types.colg import GOLD_REGION_CODES
|
||||||
from api.utils import thirdpart_gold as gold
|
from api.utils import thirdpart_gold as gold
|
||||||
|
from api.utils.gold_time import refresh_slot
|
||||||
|
|
||||||
|
|
||||||
def offer(price, url='listing', platform=17, money=1):
|
def offer(price, url='listing', platform=17, money=10):
|
||||||
return {'Price': price, 'Money': money, 'Number': 1, 'PfId': platform, 'BuyUrl': url}
|
return {'Price': price, 'Money': money, 'Number': 1, 'PfId': platform, 'BuyUrl': url}
|
||||||
|
|
||||||
|
|
||||||
def cached_item(region='跨1', age=0):
|
def cached_item(region='跨1', age=0):
|
||||||
fetched_at = time.time() - age
|
fetched_at = time.time() - age
|
||||||
return {'region': region, 'goldPrice': 100.0, 'sampleCount': 10,
|
return {'region': region, 'goldPrice': .01, 'sampleCount': 7,
|
||||||
'code': GOLD_REGION_CODES[region], 'updataTime': int(fetched_at),
|
'code': GOLD_REGION_CODES[region], 'updataTime': refresh_slot(fetched_at),
|
||||||
'fetchedAt': fetched_at}
|
'fetchedAt': fetched_at}
|
||||||
|
|
||||||
|
|
||||||
@@ -53,22 +54,28 @@ class FakeRedis:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def test_average_is_mean_of_ratios_without_minimum_money_filter():
|
def test_average_uses_displayed_unit_prices_at_indices_three_through_nine():
|
||||||
# 100 and 50 万/元 => 75; inverse mean price would incorrectly give 66.67.
|
rows = [offer(i / 100) for i in range(1, 12)]
|
||||||
assert gold.first_page_average([offer(.01), offer(.02)]) == (75.0, 2)
|
assert gold.first_page_average(rows) == (.07, 7)
|
||||||
invalid = [offer(0), offer(-1), offer(float('nan')), offer(float('inf')),
|
invalid = [offer(0), offer(-1), offer(float('nan')), offer(float('inf')),
|
||||||
offer(.01, money=-1), offer(None), offer(.01, money=float('nan'))]
|
offer(.01, money=-1), offer(None), offer(.01, money=float('nan'))]
|
||||||
assert gold.first_page_average(invalid + [offer(.01, money=.5)]) == (100.0, 1)
|
# 页面允许Money=-1的库存占位报价;另行排除不显示的低额订单。
|
||||||
|
invalid.pop(4)
|
||||||
|
assert gold.first_page_average(invalid + [offer(.01, money=.5)] + rows) == (.07, 7)
|
||||||
|
assert gold.first_page_average([offer(.016789)] * 10) == (.02, 7)
|
||||||
|
assert gold.first_page_average(rows[:4]) == (.04, 1)
|
||||||
with pytest.raises(ValueError, match='有效报价'):
|
with pytest.raises(ValueError, match='有效报价'):
|
||||||
gold.first_page_average(invalid)
|
gold.first_page_average(invalid)
|
||||||
|
with pytest.raises(ValueError, match='有效报价'):
|
||||||
|
gold.first_page_average(rows[:3])
|
||||||
|
|
||||||
|
|
||||||
def test_page_deduplicates_before_cutoff_and_preserves_7881_variants():
|
def test_page_deduplicates_before_cutoff_and_preserves_7881_variants():
|
||||||
rows = [offer(.01, 'duplicate', 40), offer(.01, 'duplicate', 40)]
|
rows = [offer(.01, 'duplicate', 40), offer(.01, 'duplicate', 40)]
|
||||||
rows += [offer(.02, str(i), 17) for i in range(199)]
|
rows += [offer(i / 100, str(i), 17) for i in range(2, 12)]
|
||||||
rows += [offer(.5, 'outside', 17)]
|
assert gold.first_page_average(rows) == (.07, 7)
|
||||||
assert gold.first_page_average(rows) == (50.25, 200)
|
rows = [offer(.01, platform=44)] * 3 + [offer(.04, platform=44)] * 7
|
||||||
assert gold.first_page_average([offer(.01, platform=44), offer(.02, platform=44)]) == (75.0, 2)
|
assert gold.first_page_average(rows) == (.04, 7)
|
||||||
|
|
||||||
|
|
||||||
def test_request_signature_matches_observed_browser_request():
|
def test_request_signature_matches_observed_browser_request():
|
||||||
@@ -118,14 +125,20 @@ def test_region_fetches_its_own_seed_config_and_all_initial_channels():
|
|||||||
|
|
||||||
def post(self, url, json, **kwargs):
|
def post(self, url, json, **kwargs):
|
||||||
posts.append((url, json))
|
posts.append((url, json))
|
||||||
return FakeResponse('{"success":1,"data":[{"Price":0.01,"Money":1,"Number":1}]}')
|
return FakeResponse('{"success":1,"data":' + json_module.dumps([
|
||||||
|
{'Price': .01, 'Money': 10, 'Number': 1} for _ in range(10)
|
||||||
|
]) + '}')
|
||||||
|
|
||||||
result = asyncio.run(gold.fetch_region(Session(), 'kua8'))
|
json_module = json
|
||||||
|
started_at = 1793766590 # 固定开始时间,与响应读取完成的当前时间不同。
|
||||||
|
result = asyncio.run(gold.fetch_region(Session(), 'kua8', started_at=started_at))
|
||||||
assert result['region'] == '跨8'
|
assert result['region'] == '跨8'
|
||||||
assert result['code'] == 'k8'
|
assert result['code'] == 'k8'
|
||||||
assert isinstance(result['updataTime'], int)
|
assert isinstance(result['updataTime'], int)
|
||||||
assert result['goldPrice'] == 100
|
assert result['goldPrice'] == .01
|
||||||
assert result['sampleCount'] == 1
|
assert result['sampleCount'] == 7
|
||||||
|
assert result['updataTime'] == refresh_slot(started_at)
|
||||||
|
assert result['fetchedAt'] == started_at
|
||||||
assert gets == [gold.BASE_URL + '/bijiaqi/dnf/youxibi/kua8',
|
assert gets == [gold.BASE_URL + '/bijiaqi/dnf/youxibi/kua8',
|
||||||
gold.BASE_URL + '/cate/1838/CoinSale_yxb_254_0.js?ver=935']
|
gold.BASE_URL + '/cate/1838/CoinSale_yxb_254_0.js?ver=935']
|
||||||
assert posts[0][1]['secret'] == hashlib.md5(b'fresh-seed_17_config-sign').hexdigest()
|
assert posts[0][1]['secret'] == hashlib.md5(b'fresh-seed_17_config-sign').hexdigest()
|
||||||
@@ -148,9 +161,25 @@ def test_cache_omits_missing_regions_marks_old_values_and_retains_order():
|
|||||||
assert [row['stale'] for row in rows] == [True, False]
|
assert [row['stale'] for row in rows] == [True, False]
|
||||||
|
|
||||||
|
|
||||||
|
def test_realtime_prefetches_at_25_and_remains_fresh_at_30(monkeypatch):
|
||||||
|
base = 1793764800 // 1800 * 1800
|
||||||
|
clock = {'now': base + 25 * 60}
|
||||||
|
monkeypatch.setattr(gold.time, 'time', lambda: clock['now'])
|
||||||
|
redis = FakeRedis()
|
||||||
|
old = cached_item(age=25 * 60)
|
||||||
|
redis.values[f'{gold.CACHE_PREFIX}:kua1'] = json.dumps(old)
|
||||||
|
assert asyncio.run(gold.read_cache(redis))[0]['stale'] is True
|
||||||
|
redis.values[f'{gold.CACHE_PREFIX}:kua1'] = json.dumps(cached_item())
|
||||||
|
prefetched = asyncio.run(gold.read_cache(redis))[0]
|
||||||
|
assert prefetched['updataTime'] == base + 30 * 60
|
||||||
|
assert prefetched['stale'] is False
|
||||||
|
clock['now'] = base + 30 * 60
|
||||||
|
assert asyncio.run(gold.read_cache(redis))[0]['stale'] is False
|
||||||
|
|
||||||
|
|
||||||
def test_invalid_cached_average_is_not_served():
|
def test_invalid_cached_average_is_not_served():
|
||||||
redis = FakeRedis()
|
redis = FakeRedis()
|
||||||
redis.values[f'{gold.CACHE_PREFIX}:kua1'] = json.dumps({**cached_item(), 'goldPrice': 0})
|
redis.values[f'{gold.CACHE_PREFIX}:kua1'] = json.dumps({**cached_item(), 'goldPrice': -1})
|
||||||
redis.values[f'{gold.CACHE_PREFIX}:kua8'] = json.dumps({**cached_item('跨8'), 'sampleCount': 0})
|
redis.values[f'{gold.CACHE_PREFIX}:kua8'] = json.dumps({**cached_item('跨8'), 'sampleCount': 0})
|
||||||
assert asyncio.run(gold.read_cache(redis)) == []
|
assert asyncio.run(gold.read_cache(redis)) == []
|
||||||
|
|
||||||
@@ -164,7 +193,7 @@ def test_legacy_cache_gets_epoch_and_region_code_without_refetching():
|
|||||||
rows = asyncio.run(gold.read_cache(redis))
|
rows = asyncio.run(gold.read_cache(redis))
|
||||||
assert len(rows) == 1
|
assert len(rows) == 1
|
||||||
assert rows[0]['code'] == 'k3a'
|
assert rows[0]['code'] == 'k3a'
|
||||||
assert rows[0]['updataTime'] == int(legacy['fetchedAt'])
|
assert rows[0]['updataTime'] == refresh_slot(legacy['fetchedAt'])
|
||||||
assert not redis.writes
|
assert not redis.writes
|
||||||
|
|
||||||
|
|
||||||
@@ -203,7 +232,7 @@ def test_refresh_preserves_failed_region_and_releases_its_lock(monkeypatch, limi
|
|||||||
monkeypatch.setattr(gold, 'REGION_INTERVAL', 0)
|
monkeypatch.setattr(gold, 'REGION_INTERVAL', 0)
|
||||||
calls = []
|
calls = []
|
||||||
|
|
||||||
async def fetch(session, region):
|
async def fetch(session, region, *, started_at=None):
|
||||||
calls.append(region)
|
calls.append(region)
|
||||||
if region == 'kua1':
|
if region == 'kua1':
|
||||||
raise gold.UpstreamLimited() if limited else ValueError('channel failed')
|
raise gold.UpstreamLimited() if limited else ValueError('channel failed')
|
||||||
@@ -357,7 +386,7 @@ def test_shutdown_cancels_refresh_and_releases_redis_lock(monkeypatch):
|
|||||||
async def run():
|
async def run():
|
||||||
started = asyncio.Event()
|
started = asyncio.Event()
|
||||||
|
|
||||||
async def fetch(session, region):
|
async def fetch(session, region, *, started_at=None):
|
||||||
started.set()
|
started.set()
|
||||||
await asyncio.Event().wait()
|
await asyncio.Event().wait()
|
||||||
|
|
||||||
@@ -382,7 +411,7 @@ def test_retry_only_fetches_missing_regions_instead_of_repeating_fresh_pages(mon
|
|||||||
monkeypatch.setattr(gold, 'create_redis', lambda: redis)
|
monkeypatch.setattr(gold, 'create_redis', lambda: redis)
|
||||||
calls = []
|
calls = []
|
||||||
|
|
||||||
async def fetch(session, region):
|
async def fetch(session, region, *, started_at=None):
|
||||||
calls.append(region)
|
calls.append(region)
|
||||||
return cached_item(gold.REGIONS[region])
|
return cached_item(gold.REGIONS[region])
|
||||||
|
|
||||||
@@ -392,6 +421,31 @@ def test_retry_only_fetches_missing_regions_instead_of_repeating_fresh_pages(mon
|
|||||||
assert len(asyncio.run(gold.read_cache(redis))) == 9
|
assert len(asyncio.run(gold.read_cache(redis))) == 9
|
||||||
|
|
||||||
|
|
||||||
|
def test_realtime_regions_share_batch_start_even_when_fetch_crosses_slot_boundary(monkeypatch):
|
||||||
|
clock = {'now': 1793766590.0}
|
||||||
|
started_at = clock['now']
|
||||||
|
redis = FakeRedis()
|
||||||
|
redis.values[gold.LOCK_KEY] = 'owner'
|
||||||
|
monkeypatch.setattr(gold, 'create_redis', lambda: redis)
|
||||||
|
monkeypatch.setattr(gold, 'REGIONS', {'kua1': '跨1', 'kua8': '跨8'})
|
||||||
|
monkeypatch.setattr(gold, 'REGION_INTERVAL', 0)
|
||||||
|
monkeypatch.setattr(gold.time, 'time', lambda: clock['now'])
|
||||||
|
starts = []
|
||||||
|
|
||||||
|
async def fetch(session, region, *, started_at):
|
||||||
|
starts.append(started_at)
|
||||||
|
clock['now'] += 1800
|
||||||
|
return {**cached_item(gold.REGIONS[region]), 'fetchedAt': started_at,
|
||||||
|
'updataTime': refresh_slot(started_at)}
|
||||||
|
|
||||||
|
monkeypatch.setattr(gold, 'fetch_region', fetch)
|
||||||
|
asyncio.run(gold.refresh_cache('owner'))
|
||||||
|
assert starts == [started_at, started_at]
|
||||||
|
timestamps = [json.loads(redis.values[f'{gold.CACHE_PREFIX}:{region}'])['updataTime']
|
||||||
|
for region in gold.REGIONS]
|
||||||
|
assert timestamps == [refresh_slot(started_at)] * 2
|
||||||
|
|
||||||
|
|
||||||
def test_unresponsive_redis_returns_prompt_error_instead_of_hanging(monkeypatch):
|
def test_unresponsive_redis_returns_prompt_error_instead_of_hanging(monkeypatch):
|
||||||
async def run():
|
async def run():
|
||||||
connections = []
|
connections = []
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
# 技能加点图片导出的跨域配置
|
||||||
|
|
||||||
|
技能树使用 `https://image.dnftools.com` 上的图标。普通 `<img>` 展示不要求 CORS,但 `html-to-image` 导出时需要 fetch 图标并内嵌到图片,因此图床必须返回允许跨域读取的响应头。前端不需要改动,`no-cors` 无法读取图片内容。
|
||||||
|
|
||||||
|
## 允许范围
|
||||||
|
|
||||||
|
仅允许 HTTP / HTTPS 的 `*.dnftools.com` 来源,包括多级子域和带端口的来源。例如 `https://dcalc.dnftools.com`、`https://test.dcalc.dnftools.com` 和 `http://dev.dnftools.com:5173`。
|
||||||
|
|
||||||
|
不包括裸域 `https://dnftools.com`、`localhost`、`null`、其他域名或 `https://dnftools.com.example.org` 这样的伪装后缀。CORS 响应头不能写成 `*.dnftools.com`,应校验 Origin 后返回该完整 Origin,并设置 `Vary: Origin` 让缓存区分来源。
|
||||||
|
|
||||||
|
## Nginx / OpenResty 图床
|
||||||
|
|
||||||
|
下面的配置适用于直接编辑站点 `server { ... }` 的环境,不需要在外层 http 中声明 map。
|
||||||
|
|
||||||
|
### 1. 在 server 层增加来源校验和默认响应头
|
||||||
|
|
||||||
|
放在 `server_name image.dnftools.com;` 后面,保留其余配置和原有的 HSTS 头:
|
||||||
|
|
||||||
|
```nginx
|
||||||
|
set $image_cors_origin "";
|
||||||
|
if ($http_origin ~* "^https?://([a-z0-9-]+\.)+dnftools\.com(:[0-9]+)?$") {
|
||||||
|
set $image_cors_origin $http_origin;
|
||||||
|
}
|
||||||
|
|
||||||
|
add_header Access-Control-Allow-Origin $image_cors_origin always;
|
||||||
|
add_header Vary "Origin" always;
|
||||||
|
```
|
||||||
|
|
||||||
|
未匹配的来源对应空值,Nginx 不输出 Access-Control-Allow-Origin。现有 `location /` 没有自己的 add_header,会继承这里的响应头。
|
||||||
|
|
||||||
|
### 2. 替换静态资源 location
|
||||||
|
|
||||||
|
这个 location 已有自己的 add_header,所以必须在这里也设置 CORS 和 Vary,不能只依赖 server 层。保留原来的文件路径、缓存、HSTS 和字符集:
|
||||||
|
|
||||||
|
```nginx
|
||||||
|
location ~* \.(?:jpg|jpeg|png|gif|webp|avif|svg|ico|bmp|swf|css|js|woff|woff2|ttf|eot)$ {
|
||||||
|
root /dcalc/image;
|
||||||
|
expires 7d;
|
||||||
|
add_header Cache-Control "public, max-age=604800";
|
||||||
|
add_header Strict-Transport-Security "max-age=31536000";
|
||||||
|
|
||||||
|
add_header Access-Control-Allow-Origin $image_cors_origin always;
|
||||||
|
add_header Vary "Origin" always;
|
||||||
|
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||||
|
add_header Access-Control-Allow-Headers "If-None-Match, If-Modified-Since" always;
|
||||||
|
add_header Access-Control-Max-Age "86400" always;
|
||||||
|
|
||||||
|
if ($request_method = OPTIONS) {
|
||||||
|
return 204;
|
||||||
|
}
|
||||||
|
|
||||||
|
try_files $uri =404;
|
||||||
|
charset utf-8,gbk;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`always` 使 CORS 头也适用于 304 和错误响应;原配置的 `error_page 404 /404.html` 内部跳转会由 `location /` 继承 server 层的 CORS 和 Vary 头。其余 location、证书和 HTTP 跳转配置保持不变。不要再保留之前的 `Access-Control-Allow-Origin "*"`,也不要重复添加此头。
|
||||||
|
|
||||||
|
这是对浏览器跨域脚本读取的限制,不是防盗链或访问鉴权;其他来源仍可能展示普通 img,或直接下载公开图片。当前导出无需 Cookie,不需要 Access-Control-Allow-Credentials。
|
||||||
|
|
||||||
|
### 3. 检查、重载及验证
|
||||||
|
|
||||||
|
在实际运行 Nginx / OpenResty 的环境中执行(容器部署则在对应容器内执行):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
nginx -t
|
||||||
|
nginx -s reload
|
||||||
|
|
||||||
|
# 应返回 Access-Control-Allow-Origin: https://dcalc.dnftools.com
|
||||||
|
# 并包含 Vary: Origin
|
||||||
|
curl -I -H 'Origin: https://dcalc.dnftools.com' \
|
||||||
|
'https://image.dnftools.com/characters/gunner_female/spitfire_female/profile.png'
|
||||||
|
|
||||||
|
# 不应返回 Access-Control-Allow-Origin
|
||||||
|
curl -I -H 'Origin: https://dnftools.com.example.org' \
|
||||||
|
'https://image.dnftools.com/characters/gunner_female/spitfire_female/profile.png'
|
||||||
|
```
|
||||||
|
|
||||||
|
如果前面还有 CDN,需让其遵守 Vary / 按 Origin 区分缓存,或在边缘执行相同的来源校验,并刷新旧响应头缓存。最后回到技能设置页导出,检查 PNG 中包含图标、等级和觉醒绑定。
|
||||||
|
|
||||||
|
## 仓库自带的 Node 图片服务
|
||||||
|
|
||||||
|
`pnpm run dev:image` 使用 `scripts/image.js`,已采用同样的来源规则:匹配时返回请求 Origin,其他来源不返回 Access-Control-Allow-Origin;所有响应设置 Vary: Origin,保留 HEAD、304、错误响应和 OPTIONS 支持。
|
||||||
|
|
||||||
|
本地前端通过 Vite 的 `/image` 同源代理访问图片,不需要放行 localhost。部署使用 Node 服务时需要重启该进程;当前生产配置直接通过 Nginx 读取 `/dcalc/image`,所以生产生效点是上面的 Nginx 修改,无需重新构建前端。
|
||||||
|
|
||||||
|
参考:[CORS 来源与缓存](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Access-Control-Allow-Origin)、[Nginx 响应头继承](https://nginx.org/en/docs/http/ngx_http_headers_module.html)、[Nginx set / if](https://nginx.org/en/docs/http/ngx_http_rewrite_module.html)。
|
||||||
+20
-1
@@ -12,6 +12,8 @@ const browserCacheSeconds = Number.parseInt(
|
|||||||
process.env.IMAGE_BROWSER_CACHE_SECONDS || '3600',
|
process.env.IMAGE_BROWSER_CACHE_SECONDS || '3600',
|
||||||
10,
|
10,
|
||||||
);
|
);
|
||||||
|
// Match complete origins, not substrings such as dnftools.com.example.org.
|
||||||
|
const allowedImageOrigin = /^https?:\/\/([a-z0-9-]+\.)+dnftools\.com(:[0-9]+)?$/i;
|
||||||
|
|
||||||
const mimeTypes = {
|
const mimeTypes = {
|
||||||
'.avif': 'image/avif',
|
'.avif': 'image/avif',
|
||||||
@@ -123,8 +125,25 @@ function sendText(res, statusCode, message) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const ImageServer = http.createServer(async (req, res) => {
|
const ImageServer = http.createServer(async (req, res) => {
|
||||||
|
// Exporting DOM to an image fetches icons across origins. Reflect only trusted
|
||||||
|
// origins, including on cached (304) and error responses.
|
||||||
|
res.setHeader('Vary', 'Origin');
|
||||||
|
const origin = req.headers.origin;
|
||||||
|
if (origin && allowedImageOrigin.test(origin)) {
|
||||||
|
res.setHeader('Access-Control-Allow-Origin', origin);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (req.method === 'OPTIONS') {
|
||||||
|
res.setHeader('Access-Control-Allow-Methods', 'GET, HEAD, OPTIONS');
|
||||||
|
res.setHeader('Access-Control-Allow-Headers', 'If-None-Match, If-Modified-Since');
|
||||||
|
res.setHeader('Access-Control-Max-Age', '86400');
|
||||||
|
res.writeHead(204);
|
||||||
|
res.end();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (req.method !== 'GET' && req.method !== 'HEAD') {
|
if (req.method !== 'GET' && req.method !== 'HEAD') {
|
||||||
res.setHeader('Allow', 'GET, HEAD');
|
res.setHeader('Allow', 'GET, HEAD, OPTIONS');
|
||||||
sendText(res, 405, 'Method not allowed');
|
sendText(res, 405, 'Method not allowed');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user