dnftools/dcalc
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Activity

update

Browse Source
This commit is contained in:
imelon
2026-11-04 15:35:15 +08:00
parent 4709ea4328
commit 62291c57ec
12 changed files with 363 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/api/utils/gold_refresh.py
+3 -1
View File
@@ -2,6 +2,7 @@
import asyncio
import logging
import time
from api.utils import official_gold, thirdpart_gold
@@ -27,7 +28,8 @@ async def _periodic(job):
await job()
except Exception:
logger.exception('periodic gold refresh check failed job=%s', job.__name__)
await asyncio.sleep(CHECK_INTERVAL_SECONDS)
# 对齐墙钟分钟边界，:25/:55检测到下一半小时数据周期并提前采集。
await asyncio.sleep(CHECK_INTERVAL_SECONDS - time.time() % CHECK_INTERVAL_SECONDS)
async def start_periodic_refresh():
backend/api/utils/gold_time.py
+18
View File
@@ -0,0 +1,18 @@
"""每个半小时刻度提前5分钟采集，返回值标记对应的半小时刻度。"""
import time
HALF_HOUR_SECONDS = 30 * 60
UPDATE_LEAD_SECONDS = 5 * 60
def half_hour_start(timestamp: float | None = None) -> int:
if timestamp is None:
timestamp = time.time()
return int(timestamp) // HALF_HOUR_SECONDS * HALF_HOUR_SECONDS
def refresh_slot(timestamp: float | None = None) -> int:
if timestamp is None:
timestamp = time.time()
return half_hour_start(timestamp + UPDATE_LEAD_SECONDS)
backend/api/utils/official_gold.py
+13 -10
View File
@@ -1,7 +1,6 @@
"""官方寄售缓存、认证失败识别及可选BARK通知。"""
import asyncio
from datetime import datetime, timezone
import hashlib
import json
import logging
@@ -14,6 +13,7 @@ import aiohttp
from api.core.Gzip import gzip_unzip, gzip_zip
from api.types.colg import GOLD_REGION_CODES, OfficialGold
from api.utils.thirdpart_gold import RELEASE_LOCK
from api.utils.gold_time import UPDATE_LEAD_SECONDS, half_hour_start, refresh_slot
from config.main import config
logger = logging.getLogger('dcalc.official_gold')
@@ -43,14 +43,14 @@ def normalize_row(item: dict) -> dict:
except (TypeError, ValueError, OverflowError):
continue
if math.isfinite(price):
prices.append(price)
prices.append(round(price, 2))
ranks[name] = prices
ratios = [10000 / price for prices in ranks.values() for price in prices
if price > 0 and math.isfinite(10000 / price)]
return OfficialGold.model_validate({
**item, **ranks,
'max': round(max(ratios), 4) if ratios else None,
'min': round(min(ratios), 4) if ratios else None,
**item, **ranks, 'updataTime': str(half_hour_start(int(item['updataTime']))),
'max': round(max(ratios), 2) if ratios else None,
'min': round(min(ratios), 2) if ratios else None,
}).model_dump()
@@ -64,7 +64,10 @@ def is_auth_failure(payload: dict) -> bool:
))
async def fetch_gold() -> list[dict]:
async def fetch_gold(*, started_at: float | None = None) -> list[dict]:
if started_at is None:
started_at = time.time()
update_time = str(refresh_slot(started_at))
timeout = aiohttp.ClientTimeout(total=10)
async with aiohttp.ClientSession(timeout=timeout) as session:
async def fetch_region(region):
@@ -81,7 +84,7 @@ async def fetch_gold() -> list[dict]:
resp.raise_for_status()
data = payload['data']
return normalize_row({
'updataTime': str(int(datetime.now(timezone.utc).timestamp())),
'updataTime': update_time,
'region': f'跨{region}', 'code': GOLD_REGION_CODES[f'跨{region}'],
'goldDealRank': [float(item['price']) for item in data['godDealRank'][:5]],
'goldTopDealRank': [float(item['price']) for item in data['godTopDealRank'][:5]],
@@ -150,7 +153,7 @@ async def read_cache(redis) -> list[dict]:
for item in json.loads(gzip_unzip(value))]
if [row['region'] for row in rows] != [f'跨{region}' for region in REGIONS]:
return []
if any(not 0 <= time.time() - int(row['updataTime']) < RETENTION_SECONDS for row in rows):
if any(not -UPDATE_LEAD_SECONDS <= time.time() - int(row['updataTime']) < RETENTION_SECONDS for row in rows):
return []
return rows
except (ValueError, TypeError, KeyError, OSError):
@@ -160,7 +163,7 @@ async def read_cache(redis) -> list[dict]:
def is_fresh(rows: list[dict]) -> bool:
return (len(rows) == len(REGIONS)
and all(0 <= time.time() - int(row['updataTime']) < FRESH_SECONDS for row in rows))
and all(int(row['updataTime']) == refresh_slot() for row in rows))
async def refresh_cache(redis) -> bool:
@@ -173,7 +176,7 @@ async def refresh_cache(redis) -> bool:
if is_fresh(await read_cache(redis)) or await redis.get(COOLDOWN_KEY):
return False
try:
data = await fetch_gold()
data = await fetch_gold(started_at=time.time())
except TokenExpired:
logger.warning('official gold authentication failed; check HELPER_TOKEN and HELPER_ID')
await notify_token_expired(redis)
backend/api/utils/thirdpart_gold.py
+24 -13
View File
@@ -9,7 +9,6 @@ import math
import re
from statistics import fmean
import time
from datetime import datetime, timedelta, timezone
from urllib.parse import urljoin, urlsplit
from uuid import uuid4
@@ -19,6 +18,7 @@ from redis.asyncio.retry import Retry
from redis.backoff import NoBackoff
from api.types.colg import GOLD_REGION_CODES, ThirdPartGoldCache
from api.utils.gold_time import refresh_slot
from config.main import config
logger = logging.getLogger('dcalc.thirdpart_gold')
@@ -36,6 +36,7 @@ HEADERS = {
# 网站公开 DNF 页面已停用/移除的渠道；不请求 VIP 专属渠道。
DISABLED_PLATFORMS = {1, 12, 21, 50, 51, 52, 53}
PAGE_SIZE = 200
SAMPLE_SLICE = slice(3, 10)
FRESH_SECONDS = 30 * 60
RETENTION_SECONDS = 24 * 60 * 60
# 按每分钟最多8次页面访问处理，理论间隔7.5秒，取8秒留出边界余量。
@@ -44,7 +45,8 @@ REGION_INTERVAL = 8
LOCK_SECONDS = 15 * 60
ERROR_COOLDOWN_SECONDS = 5 * 60
REDIS_TIMEOUT_SECONDS = 1
CACHE_PREFIX = 'openapi:colg:thirdpart_gold:v1'
# 统计范围和单位已改变，旧缓存的万金币/元均值不能作为新的元/万金币单价返回。
CACHE_PREFIX = 'openapi:colg:thirdpart_gold:v2'
LOCK_KEY = f'{CACHE_PREFIX}:lock'
COOLDOWN_KEY = f'{CACHE_PREFIX}:cooldown'
RELEASE_LOCK = """
@@ -126,7 +128,7 @@ def make_payload(channel: dict, seed: dict, user: dict) -> dict:
def first_page_average(offers: list[dict]) -> tuple[float, int]:
"""等权平均每条报价的 1/Price（万金币/元），不是 1/平均单价。"""
"""匹配公开页面 i.item1 的第4～10条，平均显示单价（元/万金币）。"""
valid = []
for offer in offers:
try:
@@ -135,9 +137,12 @@ def first_page_average(offers: list[dict]) -> tuple[float, int]:
number = float(offer['Number'])
except (KeyError, TypeError, ValueError, OverflowError):
continue
if not all(math.isfinite(value) and value > 0 for value in (price, money, number)):
if not all(math.isfinite(value) for value in (price, money, number)) or price < 0:
continue
if not math.isfinite(1 / price):
# 页面默认展示条件，不额外设置price1/price2价格过滤。
if money != -1 and (money * number < 5 or price == 0):
continue
if money >= 1_000_000 or (offer.get('Seller') or '').startswith('智莹科技'):
continue
valid.append({**offer, 'Price': price})
valid.sort(key=lambda item: (item['Price'], item['PfId'], item.get('BuyUrl') or ''))
@@ -152,9 +157,12 @@ def first_page_average(offers: list[dict]) -> tuple[float, int]:
page.append(offer)
if len(page) == PAGE_SIZE:
break
if not page:
selected = page[SAMPLE_SLICE]
if not selected:
raise ValueError('YXDR 未返回有效报价')
return round(fmean(1 / item['Price'] for item in page), 4), len(page)
# 页面 Math.floorDigits(price, 5) 先toFixed(6)再删掉末位。
prices = [float(f"{item['Price']:.6f}"[:-1]) for item in selected]
return round(fmean(prices), 2), len(prices)
async def _read_response(resp) -> str:
@@ -165,7 +173,9 @@ async def _read_response(resp) -> str:
return text
async def fetch_region(session: aiohttp.ClientSession, region: str) -> dict:
async def fetch_region(session: aiohttp.ClientSession, region: str, *, started_at: float | None = None) -> dict:
if started_at is None:
started_at = time.time()
page_url = f'{BASE_URL}/bijiaqi/dnf/youxibi/{region}'
async with session.get(page_url) as resp:
page = await _read_response(resp)
@@ -210,10 +220,9 @@ async def fetch_region(session: aiohttp.ClientSession, region: str) -> dict:
if isinstance(result, BaseException):
raise result
price, count = first_page_average([offer for result in results for offer in result])
now = datetime.now(timezone(timedelta(hours=8)))
return {'region': REGIONS[region], 'code': GOLD_REGION_CODES[REGIONS[region]],
'goldPrice': price, 'sampleCount': count,
'updataTime': int(now.timestamp()), 'fetchedAt': now.timestamp()}
'updataTime': refresh_slot(started_at), 'fetchedAt': started_at}
async def read_cache(redis) -> list[dict]:
@@ -229,8 +238,9 @@ async def read_cache(redis) -> list[dict]:
continue
# 兼容此前缓存的格式化日期及缺少code的记录，无须重新抓取所有跨区。
result.append(ThirdPartGoldCache.model_validate({
**item, 'updataTime': int(item['fetchedAt']),
'code': GOLD_REGION_CODES[REGIONS[region]], 'stale': age >= FRESH_SECONDS,
**item, 'updataTime': refresh_slot(item['fetchedAt']),
'code': GOLD_REGION_CODES[REGIONS[region]],
'stale': refresh_slot(item['fetchedAt']) < refresh_slot(),
}).model_dump())
except (KeyError, TypeError, ValueError):
logger.warning('invalid thirdpart gold cache region=%s', region)
@@ -270,6 +280,7 @@ async def refresh_cache(token: str):
fresh_regions = {item['region'] for item in cached if not item['stale']}
pending = [region for region, name in REGIONS.items() if name not in fresh_regions]
successful, last_start = 0, None
started_at = time.time()
timeout = aiohttp.ClientTimeout(total=15)
async with aiohttp.ClientSession(headers=HEADERS, timeout=timeout,
connector=aiohttp.TCPConnector(limit=5)) as session:
@@ -281,7 +292,7 @@ async def refresh_cache(token: str):
return
last_start = time.monotonic()
try:
item = await fetch_region(session, region)
item = await fetch_region(session, region, started_at=started_at)
if await redis.get(LOCK_KEY) not in (token, token.encode()):
return
await redis.set(f'{CACHE_PREFIX}:{region}', json.dumps(item),