This commit is contained in:
2026-11-04 15:35:15 +08:00
parent 4709ea4328
commit 62291c57ec
12 changed files with 363 additions and 72 deletions
+12 -3
View File
@@ -32,9 +32,17 @@ uv run ruff check .
## Gold Cache and Notifications
- Application startup checks both realtime and official gold caches, then checks
them every minute. Data older than 30 minutes refreshes without HTTP traffic.
them at wall-clock minute boundaries. Each half-hour batch starts 5 minutes
early: 01:25 collects the 01:30 batch, 01:55 collects the 02:00 batch, without
HTTP traffic. All regions in a refresh share the batch timestamp selected at
the start, even if the work crosses a half-hour boundary. Existing public
`updataTime` types remain number for realtime and string for official.
Redis locks prevent duplicate work across workers. Realtime region pages stay
8 seconds apart. Failures retain the last successful data for up to 24 hours.
- Realtime uses the arithmetic mean of displayed unit prices from rows 4–10
(`i.item1`, slice `[3:10]`), in 元/万金币, rounded to 2 decimals. This replaces
the earlier 万金币/元 average and uses a new Redis namespace so old units are
never returned as new prices. No BeautifulSoup or browser dependency is added.
- Official authentication uses `HELPER_TOKEN` and `HELPER_ID`. Authentication
failures (`returnCode=-30003`, explicit token-expiry messages or HTTP 401) can
notify via Bark. Network errors do not trigger token-expiry notifications.
@@ -46,8 +54,9 @@ uv run ruff check .
included in notification text or logged push errors.
- Official gold responses also contain `max` and `min`: the largest/smallest
values of `10000 / price` across both rank arrays, in 万金币 per 100点券,
rounded to 4 decimals. Nonpositive/invalid prices are excluded; absent valid
quotes yield `null`. Existing rank arrays retain their 点券 per 100万金币 unit.
rounded to 2 decimals. Nonpositive/invalid prices are excluded; absent valid
quotes yield `null`. Rank arrays retain their 点券 per 100万金币 unit and are
also rounded to 2 decimals. JSON numbers do not force trailing zeroes.
## Compatibility Rules
+6 -6
View File
@@ -41,9 +41,9 @@ def get_date_str(timestamp: int) -> str:
@router.get('/gold/realtime/', operation_id='realtimeGold', response_model=Response[list[RealTimeGold]])
async def get_real_time_gold(redis: Annotated[AsyncRedis, Depends(thirdpart_gold.get_redis)], background_tasks: BackgroundTasks) -> Response[list[RealTimeGold]]:
"""获取网站金币比例,数据为各跨区首次报价前200条的平均万金币/元。
"""获取网站金币单价,取页面第4~10条单价均值,元/万金币,两位小数。
updataTime为数字秒级时间戳,code为跨区标识。与thirdpart接口共用Redis缓存和
updataTime为数据所属半小时刻度的数字秒级时间戳,:25/:55提前采集对应:30/:00数据。与thirdpart接口共用Redis缓存和
后台刷新任务:30分钟自动更新,也支持按需补齐,逐区间隔8秒;初始化未齐时响应体code=202。
"""
return await get_thirdpart_gold(redis, background_tasks)
@@ -145,11 +145,11 @@ async def get_official_gold(redis: Annotated[AsyncRedis, Depends(thirdpart_gold.
@router.get('/gold/thirdpart/', operation_id='thirdpartGold', response_model=Response[list[RealTimeGold]])
async def get_thirdpart_gold(redis: Annotated[AsyncRedis, Depends(thirdpart_gold.get_redis)], background_tasks: BackgroundTasks) -> Response[list[RealTimeGold]]:
"""获取跨1~跨8(含跨3A/3B)的第三方首批报价均值,单位:万金币/元。
"""获取跨1~跨8(含跨3A/3B)的第三方单价均值,单位:元/万金币。
不限制最低总价;各平台首次返回的有效报价汇总、去重、按单价取前200条,
对每条报价的1/Price取算术平均,不按库存或份数加权。保留上游原始价格,
不额外计算交易手续费。该接口无上游分页参数,前200条是本接口的样本口径。
不额外限制最低总价;汇总渠道报价并按公开页面展示规则排序、去重,
对i.item1的第4~10条(Python切片[3:10])显示单价取算术平均,
四舍五入至两位小数,不按库存或份数加权。
每30分钟自动后台刷新,跨区页面最短间隔8秒(每分钟不超过8次),Redis锁避免重复刷新。
初始化跨区未齐时响应体code=202并注明进度;9区缓存齐全才返回code=200。
+32 -6
View File
@@ -1,4 +1,6 @@
from pydantic import BaseModel, Field
from pydantic import BaseModel, Field, field_validator
from api.utils.gold_time import half_hour_start
GOLD_REGION_CODES = {
@@ -8,20 +10,29 @@ GOLD_REGION_CODES = {
class RealTimeGold(BaseModel):
updataTime: int = Field(..., ge=0, description='更新时间,Unix秒级时间戳')
updataTime: int = Field(..., ge=0, description='数据所属半小时刻度的Unix秒级时间戳,提前5分钟开始采集')
"""更新时间"""
region: str = Field(..., description='跨区')
"""跨区"""
code: str = Field(..., description='跨区code:k1、k2、k3a、k3b、k4、k5、k6、k7、k8')
goldPrice: float = Field(..., gt=0, allow_inf_nan=False, description='金币比例,1元对应的万金币数量')
goldPrice: float = Field(..., ge=0, allow_inf_nan=False, description='页面第4~10条单价均值,元/万金币,四舍五入至两位小数')
"""金价"""
@field_validator('updataTime')
@classmethod
def align_update_time(cls, value):
return half_hour_start(value)
@field_validator('goldPrice')
@classmethod
def round_gold_price(cls, value):
return round(value, 2)
class ThirdPartGoldCache(RealTimeGold):
"""内部缓存数据,不作为接口返回模型。"""
goldPrice: float = Field(..., gt=0, allow_inf_nan=False, description='首页有效报价换算为万金币/元后的算术平均值')
sampleCount: int = Field(..., ge=1, le=200, description='参与均值计算的报价条数')
sampleCount: int = Field(..., ge=1, le=7, description='第4~10条中参与均值计算的报价条数')
stale: bool = Field(..., description='数据是否已超过30分钟刷新周期')
@@ -41,7 +52,7 @@ class HistoryGold(BaseModel):
"""echarts数据"""
class OfficialGold(BaseModel):
updataTime: str = Field(..., pattern=r'^\d+$', description='更新时间,Unix秒级时间戳字符串')
updataTime: str = Field(..., pattern=r'^\d+$', description='数据所属半小时刻度的Unix秒级时间戳字符串,提前5分钟开始采集')
"""更新时间"""
region: str = Field(..., description='跨区')
"""跨区"""
@@ -52,3 +63,18 @@ class OfficialGold(BaseModel):
"""金币寄售最高成交价排名前5的价格"""
max: float | None = Field(..., description='100点券可兑换的最大万金币数量,无有效报价时为null')
min: float | None = Field(..., description='100点券可兑换的最小万金币数量,无有效报价时为null')
@field_validator('updataTime')
@classmethod
def align_update_time(cls, value):
return str(half_hour_start(int(value)))
@field_validator('goldDealRank', 'goldTopDealRank')
@classmethod
def round_ranks(cls, values):
return [round(value, 2) for value in values]
@field_validator('max', 'min')
@classmethod
def round_ratios(cls, value):
return round(value, 2) if value is not None else None
+3 -1
View File
@@ -2,6 +2,7 @@
import asyncio
import logging
import time
from api.utils import official_gold, thirdpart_gold
@@ -27,7 +28,8 @@ async def _periodic(job):
await job()
except Exception:
logger.exception('periodic gold refresh check failed job=%s', job.__name__)
await asyncio.sleep(CHECK_INTERVAL_SECONDS)
# 对齐墙钟分钟边界,:25/:55检测到下一半小时数据周期并提前采集。
await asyncio.sleep(CHECK_INTERVAL_SECONDS - time.time() % CHECK_INTERVAL_SECONDS)
async def start_periodic_refresh():
+18
View File
@@ -0,0 +1,18 @@
"""每个半小时刻度提前5分钟采集,返回值标记对应的半小时刻度。"""
import time
HALF_HOUR_SECONDS = 30 * 60
UPDATE_LEAD_SECONDS = 5 * 60
def half_hour_start(timestamp: float | None = None) -> int:
if timestamp is None:
timestamp = time.time()
return int(timestamp) // HALF_HOUR_SECONDS * HALF_HOUR_SECONDS
def refresh_slot(timestamp: float | None = None) -> int:
if timestamp is None:
timestamp = time.time()
return half_hour_start(timestamp + UPDATE_LEAD_SECONDS)
+13 -10
View File
@@ -1,7 +1,6 @@
"""官方寄售缓存、认证失败识别及可选BARK通知。"""
import asyncio
from datetime import datetime, timezone
import hashlib
import json
import logging
@@ -14,6 +13,7 @@ import aiohttp
from api.core.Gzip import gzip_unzip, gzip_zip
from api.types.colg import GOLD_REGION_CODES, OfficialGold
from api.utils.thirdpart_gold import RELEASE_LOCK
from api.utils.gold_time import UPDATE_LEAD_SECONDS, half_hour_start, refresh_slot
from config.main import config
logger = logging.getLogger('dcalc.official_gold')
@@ -43,14 +43,14 @@ def normalize_row(item: dict) -> dict:
except (TypeError, ValueError, OverflowError):
continue
if math.isfinite(price):
prices.append(price)
prices.append(round(price, 2))
ranks[name] = prices
ratios = [10000 / price for prices in ranks.values() for price in prices
if price > 0 and math.isfinite(10000 / price)]
return OfficialGold.model_validate({
**item, **ranks,
'max': round(max(ratios), 4) if ratios else None,
'min': round(min(ratios), 4) if ratios else None,
**item, **ranks, 'updataTime': str(half_hour_start(int(item['updataTime']))),
'max': round(max(ratios), 2) if ratios else None,
'min': round(min(ratios), 2) if ratios else None,
}).model_dump()
@@ -64,7 +64,10 @@ def is_auth_failure(payload: dict) -> bool:
))
async def fetch_gold() -> list[dict]:
async def fetch_gold(*, started_at: float | None = None) -> list[dict]:
if started_at is None:
started_at = time.time()
update_time = str(refresh_slot(started_at))
timeout = aiohttp.ClientTimeout(total=10)
async with aiohttp.ClientSession(timeout=timeout) as session:
async def fetch_region(region):
@@ -81,7 +84,7 @@ async def fetch_gold() -> list[dict]:
resp.raise_for_status()
data = payload['data']
return normalize_row({
'updataTime': str(int(datetime.now(timezone.utc).timestamp())),
'updataTime': update_time,
'region': f'跨{region}', 'code': GOLD_REGION_CODES[f'跨{region}'],
'goldDealRank': [float(item['price']) for item in data['godDealRank'][:5]],
'goldTopDealRank': [float(item['price']) for item in data['godTopDealRank'][:5]],
@@ -150,7 +153,7 @@ async def read_cache(redis) -> list[dict]:
for item in json.loads(gzip_unzip(value))]
if [row['region'] for row in rows] != [f'跨{region}' for region in REGIONS]:
return []
if any(not 0 <= time.time() - int(row['updataTime']) < RETENTION_SECONDS for row in rows):
if any(not -UPDATE_LEAD_SECONDS <= time.time() - int(row['updataTime']) < RETENTION_SECONDS for row in rows):
return []
return rows
except (ValueError, TypeError, KeyError, OSError):
@@ -160,7 +163,7 @@ async def read_cache(redis) -> list[dict]:
def is_fresh(rows: list[dict]) -> bool:
return (len(rows) == len(REGIONS)
and all(0 <= time.time() - int(row['updataTime']) < FRESH_SECONDS for row in rows))
and all(int(row['updataTime']) == refresh_slot() for row in rows))
async def refresh_cache(redis) -> bool:
@@ -173,7 +176,7 @@ async def refresh_cache(redis) -> bool:
if is_fresh(await read_cache(redis)) or await redis.get(COOLDOWN_KEY):
return False
try:
data = await fetch_gold()
data = await fetch_gold(started_at=time.time())
except TokenExpired:
logger.warning('official gold authentication failed; check HELPER_TOKEN and HELPER_ID')
await notify_token_expired(redis)
+24 -13
View File
@@ -9,7 +9,6 @@ import math
import re
from statistics import fmean
import time
from datetime import datetime, timedelta, timezone
from urllib.parse import urljoin, urlsplit
from uuid import uuid4
@@ -19,6 +18,7 @@ from redis.asyncio.retry import Retry
from redis.backoff import NoBackoff
from api.types.colg import GOLD_REGION_CODES, ThirdPartGoldCache
from api.utils.gold_time import refresh_slot
from config.main import config
logger = logging.getLogger('dcalc.thirdpart_gold')
@@ -36,6 +36,7 @@ HEADERS = {
# 网站公开 DNF 页面已停用/移除的渠道;不请求 VIP 专属渠道。
DISABLED_PLATFORMS = {1, 12, 21, 50, 51, 52, 53}
PAGE_SIZE = 200
SAMPLE_SLICE = slice(3, 10)
FRESH_SECONDS = 30 * 60
RETENTION_SECONDS = 24 * 60 * 60
# 按每分钟最多8次页面访问处理,理论间隔7.5秒,取8秒留出边界余量。
@@ -44,7 +45,8 @@ REGION_INTERVAL = 8
LOCK_SECONDS = 15 * 60
ERROR_COOLDOWN_SECONDS = 5 * 60
REDIS_TIMEOUT_SECONDS = 1
CACHE_PREFIX = 'openapi:colg:thirdpart_gold:v1'
# 统计范围和单位已改变,旧缓存的万金币/元均值不能作为新的元/万金币单价返回。
CACHE_PREFIX = 'openapi:colg:thirdpart_gold:v2'
LOCK_KEY = f'{CACHE_PREFIX}:lock'
COOLDOWN_KEY = f'{CACHE_PREFIX}:cooldown'
RELEASE_LOCK = """
@@ -126,7 +128,7 @@ def make_payload(channel: dict, seed: dict, user: dict) -> dict:
def first_page_average(offers: list[dict]) -> tuple[float, int]:
"""等权平均每条报价的 1/Price(万金币/元),不是 1/平均单价。"""
"""匹配公开页面 i.item1 的第4~10条,平均显示单价(元/万金币)。"""
valid = []
for offer in offers:
try:
@@ -135,9 +137,12 @@ def first_page_average(offers: list[dict]) -> tuple[float, int]:
number = float(offer['Number'])
except (KeyError, TypeError, ValueError, OverflowError):
continue
if not all(math.isfinite(value) and value > 0 for value in (price, money, number)):
if not all(math.isfinite(value) for value in (price, money, number)) or price < 0:
continue
if not math.isfinite(1 / price):
# 页面默认展示条件,不额外设置price1/price2价格过滤。
if money != -1 and (money * number < 5 or price == 0):
continue
if money >= 1_000_000 or (offer.get('Seller') or '').startswith('智莹科技'):
continue
valid.append({**offer, 'Price': price})
valid.sort(key=lambda item: (item['Price'], item['PfId'], item.get('BuyUrl') or ''))
@@ -152,9 +157,12 @@ def first_page_average(offers: list[dict]) -> tuple[float, int]:
page.append(offer)
if len(page) == PAGE_SIZE:
break
if not page:
selected = page[SAMPLE_SLICE]
if not selected:
raise ValueError('YXDR 未返回有效报价')
return round(fmean(1 / item['Price'] for item in page), 4), len(page)
# 页面 Math.floorDigits(price, 5) 先toFixed(6)再删掉末位。
prices = [float(f"{item['Price']:.6f}"[:-1]) for item in selected]
return round(fmean(prices), 2), len(prices)
async def _read_response(resp) -> str:
@@ -165,7 +173,9 @@ async def _read_response(resp) -> str:
return text
async def fetch_region(session: aiohttp.ClientSession, region: str) -> dict:
async def fetch_region(session: aiohttp.ClientSession, region: str, *, started_at: float | None = None) -> dict:
if started_at is None:
started_at = time.time()
page_url = f'{BASE_URL}/bijiaqi/dnf/youxibi/{region}'
async with session.get(page_url) as resp:
page = await _read_response(resp)
@@ -210,10 +220,9 @@ async def fetch_region(session: aiohttp.ClientSession, region: str) -> dict:
if isinstance(result, BaseException):
raise result
price, count = first_page_average([offer for result in results for offer in result])
now = datetime.now(timezone(timedelta(hours=8)))
return {'region': REGIONS[region], 'code': GOLD_REGION_CODES[REGIONS[region]],
'goldPrice': price, 'sampleCount': count,
'updataTime': int(now.timestamp()), 'fetchedAt': now.timestamp()}
'updataTime': refresh_slot(started_at), 'fetchedAt': started_at}
async def read_cache(redis) -> list[dict]:
@@ -229,8 +238,9 @@ async def read_cache(redis) -> list[dict]:
continue
# 兼容此前缓存的格式化日期及缺少code的记录,无须重新抓取所有跨区。
result.append(ThirdPartGoldCache.model_validate({
**item, 'updataTime': int(item['fetchedAt']),
'code': GOLD_REGION_CODES[REGIONS[region]], 'stale': age >= FRESH_SECONDS,
**item, 'updataTime': refresh_slot(item['fetchedAt']),
'code': GOLD_REGION_CODES[REGIONS[region]],
'stale': refresh_slot(item['fetchedAt']) < refresh_slot(),
}).model_dump())
except (KeyError, TypeError, ValueError):
logger.warning('invalid thirdpart gold cache region=%s', region)
@@ -270,6 +280,7 @@ async def refresh_cache(token: str):
fresh_regions = {item['region'] for item in cached if not item['stale']}
pending = [region for region, name in REGIONS.items() if name not in fresh_regions]
successful, last_start = 0, None
started_at = time.time()
timeout = aiohttp.ClientTimeout(total=15)
async with aiohttp.ClientSession(headers=HEADERS, timeout=timeout,
connector=aiohttp.TCPConnector(limit=5)) as session:
@@ -281,7 +292,7 @@ async def refresh_cache(token: str):
return
last_start = time.monotonic()
try:
item = await fetch_region(session, region)
item = await fetch_region(session, region, started_at=started_at)
if await redis.get(LOCK_KEY) not in (token, token.encode()):
return
await redis.set(f'{CACHE_PREFIX}:{region}', json.dumps(item),
+36 -12
View File
@@ -6,6 +6,7 @@ import pytest
from api.utils import official_gold as gold
from api.core.Gzip import gzip_zip
from api.utils.gold_time import refresh_slot
class FakeResponse:
@@ -67,14 +68,37 @@ def test_fetch_official_gold_uses_async_http_with_isolated_forms(monkeypatch):
assert [item['region'] for item in result] == [f'跨{region}' for region in gold.REGIONS]
assert [item['code'] for item in result] == ['k1', 'k2', 'k3a', 'k3b', 'k4', 'k5', 'k6', 'k7']
assert all(isinstance(item['updataTime'], str) and item['updataTime'].isdigit() for item in result)
assert abs(int(result[0]['updataTime']) - time.time()) < 2
assert int(result[0]['updataTime']) == refresh_slot()
assert result[0]['goldDealRank'] == [100.5, 101, 102, 103, 104]
assert result[0]['goldTopDealRank'] == [200.25, 201, 202, 203, 204]
assert result[0]['max'] == round(10000 / 100.5, 4)
assert result[0]['min'] == round(10000 / 204, 4)
assert result[0]['max'] == round(10000 / 100.5, 2)
assert result[0]['min'] == round(10000 / 204, 2)
assert sessions[0].kwargs['timeout'].total == 10
def test_official_batch_timestamp_comes_from_start_not_fetch_completion(monkeypatch):
started_at = 1793766300 # 与真实结束时间不同的固定开始时间。
monkeypatch.setattr(gold.aiohttp, 'ClientSession', lambda **kwargs: FakeSession([], **kwargs))
rows = asyncio.run(gold.fetch_gold(started_at=started_at))
assert {row['updataTime'] for row in rows} == {str(refresh_slot(started_at))}
def test_preloaded_next_slot_cache_is_servable_before_the_half_hour(monkeypatch):
from datetime import datetime, timedelta, timezone
china = timezone(timedelta(hours=8))
now = datetime(2026, 11, 4, 1, 25, tzinfo=china).timestamp()
monkeypatch.setattr(gold.time, 'time', lambda: now)
rows = cached_rows()
redis = FakeRedis()
redis.values[gold.CACHE_KEY] = gzip_zip(json.dumps(rows))
assert gold.is_fresh(asyncio.run(gold.read_cache(redis)))
target = int(rows[0]['updataTime'])
assert target > now
assert target - now == 300
monkeypatch.setattr(gold.time, 'time', lambda: now + 1800)
assert not gold.is_fresh(rows)
def test_official_route_uses_new_cache_contract_and_numeric_rank_arrays(monkeypatch, client):
calls = []
@@ -82,7 +106,7 @@ def test_official_route_uses_new_cache_contract_and_numeric_rank_arrays(monkeypa
calls.append(redis)
return [{'updataTime': '1793711716', 'region': '跨3B', 'code': 'k3b',
'goldDealRank': [100.5, 101], 'goldTopDealRank': [200.25, 201],
'max': round(10000 / 100.5, 4), 'min': round(10000 / 201, 4)}]
'max': round(10000 / 100.5, 2), 'min': round(10000 / 201, 2)}]
monkeypatch.setattr(gold, 'get_data', cached)
body = client.get('/api/open/gold/official/').json()
@@ -95,8 +119,8 @@ def test_official_route_uses_new_cache_contract_and_numeric_rank_arrays(monkeypa
assert row['code'] == 'k3b'
assert row['goldDealRank'] == [100.5, 101]
assert row['goldTopDealRank'] == [200.25, 201]
assert row['max'] == round(10000 / 100.5, 4)
assert row['min'] == round(10000 / 201, 4)
assert row['max'] == round(10000 / 100.5, 2)
assert row['min'] == round(10000 / 201, 2)
for rank in (row['goldDealRank'], row['goldTopDealRank']):
assert len(rank) <= 5
assert all(isinstance(value, (int, float)) for value in rank)
@@ -140,7 +164,7 @@ class FakeRedis:
def cached_rows(age=0):
return [{'updataTime': str(int(time.time() - age)), 'region': f'跨{region}',
return [{'updataTime': str(refresh_slot(time.time() - age)), 'region': f'跨{region}',
'code': f'k{region.lower()}', 'goldDealRank': [100.5], 'goldTopDealRank': [200.25]}
for region in gold.REGIONS]
@@ -169,8 +193,8 @@ def test_old_official_cache_gets_max_and_min_without_refetch():
redis.values[gold.CACHE_KEY] = gzip_zip(json.dumps(cached_rows()))
rows = asyncio.run(gold.read_cache(redis))
assert len(rows) == 8
assert rows[0]['max'] == round(10000 / 100.5, 4)
assert rows[0]['min'] == round(10000 / 200.25, 4)
assert rows[0]['max'] == round(10000 / 100.5, 2)
assert rows[0]['min'] == round(10000 / 200.25, 2)
@pytest.mark.parametrize('payload', [
@@ -206,7 +230,7 @@ def test_refresh_updates_cache_without_http_request_and_skips_when_fresh(monkeyp
redis = FakeRedis()
calls = []
async def fetch():
async def fetch(*, started_at=None):
calls.append(1)
return cached_rows()
@@ -225,7 +249,7 @@ def test_failed_refresh_preserves_cache_and_notifies_once_per_fetch(monkeypatch)
redis.values[gold.CACHE_KEY] = old
alerts = []
async def fetch():
async def fetch(*, started_at=None):
raise gold.TokenExpired()
async def notify(redis):
@@ -247,7 +271,7 @@ def test_network_failure_preserves_cache_without_token_alert(monkeypatch):
old = gzip_zip(json.dumps(cached_rows(age=1900)))
redis.values[gold.CACHE_KEY] = old
async def fetch():
async def fetch(*, started_at=None):
raise TimeoutError('upstream unavailable')
async def notify(redis):
+38
View File
@@ -0,0 +1,38 @@
from datetime import datetime, timedelta, timezone
import pytest
from api.types.colg import OfficialGold, RealTimeGold
from api.utils.gold_time import half_hour_start, refresh_slot
CHINA = timezone(timedelta(hours=8))
def stamp(hour, minute, second=0):
return int(datetime(2026, 11, 4, hour, minute, second, tzinfo=CHINA).timestamp())
@pytest.mark.parametrize(('hour', 'minute', 'second', 'target_hour', 'target_minute'), [
(1, 24, 59, 1, 0), (1, 25, 0, 1, 30), (1, 29, 59, 1, 30),
(1, 30, 0, 1, 30), (1, 54, 59, 1, 30), (1, 55, 0, 2, 0), (2, 0, 0, 2, 0),
(23, 55, 0, 0, 0),
])
def test_refresh_slot_advances_five_minutes_before_boundary(hour, minute, second, target_hour, target_minute):
expected = stamp(target_hour, target_minute)
if hour == 23:
expected += 24 * 60 * 60
assert refresh_slot(stamp(hour, minute, second)) == expected
def test_response_types_round_price_values_and_keep_timestamp_types():
realtime = RealTimeGold(updataTime=stamp(1, 30), region='跨1', code='k1', goldPrice=.016789)
assert realtime.goldPrice == .02
assert realtime.updataTime == stamp(1, 30)
assert isinstance(realtime.updataTime, int)
official = OfficialGold(updataTime=str(stamp(1, 30)), region='跨1', code='k1',
goldDealRank=[125.556], goldTopDealRank=[250.123], max=79.64444, min=39.9876)
assert official.goldDealRank == [125.56]
assert official.goldTopDealRank == [250.12]
assert official.max == 79.64 and official.min == 39.99
assert official.updataTime == str(stamp(1, 30))
assert half_hour_start(stamp(1, 29, 59)) == stamp(1, 0)
+74 -20
View File
@@ -9,16 +9,17 @@ from fastapi import BackgroundTasks
from api.routes import colg
from api.types.colg import GOLD_REGION_CODES
from api.utils import thirdpart_gold as gold
from api.utils.gold_time import refresh_slot
def offer(price, url='listing', platform=17, money=1):
def offer(price, url='listing', platform=17, money=10):
return {'Price': price, 'Money': money, 'Number': 1, 'PfId': platform, 'BuyUrl': url}
def cached_item(region='跨1', age=0):
fetched_at = time.time() - age
return {'region': region, 'goldPrice': 100.0, 'sampleCount': 10,
'code': GOLD_REGION_CODES[region], 'updataTime': int(fetched_at),
return {'region': region, 'goldPrice': .01, 'sampleCount': 7,
'code': GOLD_REGION_CODES[region], 'updataTime': refresh_slot(fetched_at),
'fetchedAt': fetched_at}
@@ -53,22 +54,28 @@ class FakeRedis:
return 0
def test_average_is_mean_of_ratios_without_minimum_money_filter():
# 100 and 50 万/元 => 75; inverse mean price would incorrectly give 66.67.
assert gold.first_page_average([offer(.01), offer(.02)]) == (75.0, 2)
def test_average_uses_displayed_unit_prices_at_indices_three_through_nine():
rows = [offer(i / 100) for i in range(1, 12)]
assert gold.first_page_average(rows) == (.07, 7)
invalid = [offer(0), offer(-1), offer(float('nan')), offer(float('inf')),
offer(.01, money=-1), offer(None), offer(.01, money=float('nan'))]
assert gold.first_page_average(invalid + [offer(.01, money=.5)]) == (100.0, 1)
# 页面允许Money=-1的库存占位报价;另行排除不显示的低额订单。
invalid.pop(4)
assert gold.first_page_average(invalid + [offer(.01, money=.5)] + rows) == (.07, 7)
assert gold.first_page_average([offer(.016789)] * 10) == (.02, 7)
assert gold.first_page_average(rows[:4]) == (.04, 1)
with pytest.raises(ValueError, match='有效报价'):
gold.first_page_average(invalid)
with pytest.raises(ValueError, match='有效报价'):
gold.first_page_average(rows[:3])
def test_page_deduplicates_before_cutoff_and_preserves_7881_variants():
rows = [offer(.01, 'duplicate', 40), offer(.01, 'duplicate', 40)]
rows += [offer(.02, str(i), 17) for i in range(199)]
rows += [offer(.5, 'outside', 17)]
assert gold.first_page_average(rows) == (50.25, 200)
assert gold.first_page_average([offer(.01, platform=44), offer(.02, platform=44)]) == (75.0, 2)
rows += [offer(i / 100, str(i), 17) for i in range(2, 12)]
assert gold.first_page_average(rows) == (.07, 7)
rows = [offer(.01, platform=44)] * 3 + [offer(.04, platform=44)] * 7
assert gold.first_page_average(rows) == (.04, 7)
def test_request_signature_matches_observed_browser_request():
@@ -118,14 +125,20 @@ def test_region_fetches_its_own_seed_config_and_all_initial_channels():
def post(self, url, json, **kwargs):
posts.append((url, json))
return FakeResponse('{"success":1,"data":[{"Price":0.01,"Money":1,"Number":1}]}')
return FakeResponse('{"success":1,"data":' + json_module.dumps([
{'Price': .01, 'Money': 10, 'Number': 1} for _ in range(10)
]) + '}')
result = asyncio.run(gold.fetch_region(Session(), 'kua8'))
json_module = json
started_at = 1793766590 # 固定开始时间,与响应读取完成的当前时间不同。
result = asyncio.run(gold.fetch_region(Session(), 'kua8', started_at=started_at))
assert result['region'] == '跨8'
assert result['code'] == 'k8'
assert isinstance(result['updataTime'], int)
assert result['goldPrice'] == 100
assert result['sampleCount'] == 1
assert result['goldPrice'] == .01
assert result['sampleCount'] == 7
assert result['updataTime'] == refresh_slot(started_at)
assert result['fetchedAt'] == started_at
assert gets == [gold.BASE_URL + '/bijiaqi/dnf/youxibi/kua8',
gold.BASE_URL + '/cate/1838/CoinSale_yxb_254_0.js?ver=935']
assert posts[0][1]['secret'] == hashlib.md5(b'fresh-seed_17_config-sign').hexdigest()
@@ -148,9 +161,25 @@ def test_cache_omits_missing_regions_marks_old_values_and_retains_order():
assert [row['stale'] for row in rows] == [True, False]
def test_realtime_prefetches_at_25_and_remains_fresh_at_30(monkeypatch):
base = 1793764800 // 1800 * 1800
clock = {'now': base + 25 * 60}
monkeypatch.setattr(gold.time, 'time', lambda: clock['now'])
redis = FakeRedis()
old = cached_item(age=25 * 60)
redis.values[f'{gold.CACHE_PREFIX}:kua1'] = json.dumps(old)
assert asyncio.run(gold.read_cache(redis))[0]['stale'] is True
redis.values[f'{gold.CACHE_PREFIX}:kua1'] = json.dumps(cached_item())
prefetched = asyncio.run(gold.read_cache(redis))[0]
assert prefetched['updataTime'] == base + 30 * 60
assert prefetched['stale'] is False
clock['now'] = base + 30 * 60
assert asyncio.run(gold.read_cache(redis))[0]['stale'] is False
def test_invalid_cached_average_is_not_served():
redis = FakeRedis()
redis.values[f'{gold.CACHE_PREFIX}:kua1'] = json.dumps({**cached_item(), 'goldPrice': 0})
redis.values[f'{gold.CACHE_PREFIX}:kua1'] = json.dumps({**cached_item(), 'goldPrice': -1})
redis.values[f'{gold.CACHE_PREFIX}:kua8'] = json.dumps({**cached_item('跨8'), 'sampleCount': 0})
assert asyncio.run(gold.read_cache(redis)) == []
@@ -164,7 +193,7 @@ def test_legacy_cache_gets_epoch_and_region_code_without_refetching():
rows = asyncio.run(gold.read_cache(redis))
assert len(rows) == 1
assert rows[0]['code'] == 'k3a'
assert rows[0]['updataTime'] == int(legacy['fetchedAt'])
assert rows[0]['updataTime'] == refresh_slot(legacy['fetchedAt'])
assert not redis.writes
@@ -203,7 +232,7 @@ def test_refresh_preserves_failed_region_and_releases_its_lock(monkeypatch, limi
monkeypatch.setattr(gold, 'REGION_INTERVAL', 0)
calls = []
async def fetch(session, region):
async def fetch(session, region, *, started_at=None):
calls.append(region)
if region == 'kua1':
raise gold.UpstreamLimited() if limited else ValueError('channel failed')
@@ -357,7 +386,7 @@ def test_shutdown_cancels_refresh_and_releases_redis_lock(monkeypatch):
async def run():
started = asyncio.Event()
async def fetch(session, region):
async def fetch(session, region, *, started_at=None):
started.set()
await asyncio.Event().wait()
@@ -382,7 +411,7 @@ def test_retry_only_fetches_missing_regions_instead_of_repeating_fresh_pages(mon
monkeypatch.setattr(gold, 'create_redis', lambda: redis)
calls = []
async def fetch(session, region):
async def fetch(session, region, *, started_at=None):
calls.append(region)
return cached_item(gold.REGIONS[region])
@@ -392,6 +421,31 @@ def test_retry_only_fetches_missing_regions_instead_of_repeating_fresh_pages(mon
assert len(asyncio.run(gold.read_cache(redis))) == 9
def test_realtime_regions_share_batch_start_even_when_fetch_crosses_slot_boundary(monkeypatch):
clock = {'now': 1793766590.0}
started_at = clock['now']
redis = FakeRedis()
redis.values[gold.LOCK_KEY] = 'owner'
monkeypatch.setattr(gold, 'create_redis', lambda: redis)
monkeypatch.setattr(gold, 'REGIONS', {'kua1': '跨1', 'kua8': '跨8'})
monkeypatch.setattr(gold, 'REGION_INTERVAL', 0)
monkeypatch.setattr(gold.time, 'time', lambda: clock['now'])
starts = []
async def fetch(session, region, *, started_at):
starts.append(started_at)
clock['now'] += 1800
return {**cached_item(gold.REGIONS[region]), 'fetchedAt': started_at,
'updataTime': refresh_slot(started_at)}
monkeypatch.setattr(gold, 'fetch_region', fetch)
asyncio.run(gold.refresh_cache('owner'))
assert starts == [started_at, started_at]
timestamps = [json.loads(redis.values[f'{gold.CACHE_PREFIX}:{region}'])['updataTime']
for region in gold.REGIONS]
assert timestamps == [refresh_slot(started_at)] * 2
def test_unresponsive_redis_returns_prompt_error_instead_of_hanging(monkeypatch):
async def run():
connections = []