feat:添加colg对接功能
This commit is contained in:
+45
-62
@@ -1,19 +1,17 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from api.types.colg import HistoryGold, OfficialGold, RealTimeGold
|
||||
from api.utils import official_gold, thirdpart_gold
|
||||
from api.core.Gzip import register_gzip_request
|
||||
import json
|
||||
from fastapi import APIRouter, Query
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, Query
|
||||
from redis.asyncio import Redis as AsyncRedis
|
||||
from api.core.Response import response
|
||||
from api.core.Redis import get_redis_info, get_redis_info_async
|
||||
from api.dp import AsyncRedisDep, RedisDep
|
||||
from typing import Annotated
|
||||
import requests
|
||||
from config.main import config
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from api.core.Response import Return as Response
|
||||
import aiohttp
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger('dcalc.colg')
|
||||
@@ -35,36 +33,6 @@ url = 'https://poservice.colg.cn/api/v1/trend/getCoinTrendDataKol'
|
||||
|
||||
payload = {'token': config.COLG_TOKEN, 'area': 0, 'time_type': 'other', 'start_time': '2026-01-06', 'end_time': '2026-01-06'}
|
||||
|
||||
payload_helper = {
|
||||
'r': 'godRank/godDealRank',
|
||||
'userId': config.HELPER_ID,
|
||||
'token': config.HELPER_TOKEN,
|
||||
}
|
||||
|
||||
OFFICIAL_GOLD_URL = 'https://dzhu.qq.com/zangyi/activity/app'
|
||||
OFFICIAL_GOLD_REGIONS = ('1', '2', '3A', '3B', '4', '5', '6', '7')
|
||||
|
||||
|
||||
async def _fetch_official_gold() -> list[dict]:
|
||||
timeout = aiohttp.ClientTimeout(total=10)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
|
||||
async def fetch_region(region: str) -> dict:
|
||||
form = {**payload_helper, 'serverNo': region}
|
||||
async with session.post(OFFICIAL_GOLD_URL, data=form) as resp:
|
||||
resp.raise_for_status()
|
||||
payload = await resp.json(content_type=None)
|
||||
data = payload['data']
|
||||
return {
|
||||
'updataTime': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
||||
'region': f'跨{region}',
|
||||
'goldDealRank': [int(item['price']) for item in data['godDealRank']],
|
||||
'goldTopDealRank': [int(item['price']) for item in data['godTopDealRank']],
|
||||
}
|
||||
|
||||
return await asyncio.gather(*(fetch_region(region) for region in OFFICIAL_GOLD_REGIONS))
|
||||
|
||||
|
||||
def get_date_str(timestamp: int) -> str:
|
||||
utc_time = datetime.fromtimestamp(timestamp, tz=timezone.utc)
|
||||
beijing_time = utc_time + timedelta(hours=8)
|
||||
@@ -72,30 +40,13 @@ def get_date_str(timestamp: int) -> str:
|
||||
|
||||
|
||||
@router.get('/gold/realtime/', operation_id='realtimeGold', response_model=Response[list[RealTimeGold]])
|
||||
def get_real_time_gold(redis: RedisDep) -> Response[list[RealTimeGold]]:
|
||||
"""
|
||||
返回所有跨区的实时金价信息
|
||||
"""
|
||||
key = 'openapi:colg:real_time_gold'
|
||||
try:
|
||||
async def get_real_time_gold(redis: Annotated[AsyncRedis, Depends(thirdpart_gold.get_redis)], background_tasks: BackgroundTasks) -> Response[list[RealTimeGold]]:
|
||||
"""获取网站金币比例，数据为各跨区首次报价前200条的平均万金币/元。
|
||||
|
||||
def get_gold_data():
|
||||
res = []
|
||||
for region_name, region_id in regionMap.items():
|
||||
payload['area'] = region_id
|
||||
payload['start_time'] = datetime.now().strftime('%Y-%m-%d')
|
||||
payload['end_time'] = datetime.now().strftime('%Y-%m-%d')
|
||||
response = requests.post(url, headers={'Content-Type': 'application/json'}, data=json.dumps(payload))
|
||||
response.raise_for_status()
|
||||
resp = response.json().get('data', {})
|
||||
res.append({'updataTime': get_date_str(resp['x'][-1]), 'region': region_name, 'goldPrice': resp['y'][-1]})
|
||||
return res
|
||||
except Exception:
|
||||
# print(f"File not found: {e}")
|
||||
return response(data=[])
|
||||
# 设置半小时过期一次
|
||||
data = get_redis_info(redis, key, get_gold_data, 30 * 60)
|
||||
return response(data=data)
|
||||
updataTime为数字秒级时间戳，code为跨区标识。与thirdpart接口共用Redis缓存和
|
||||
后台刷新任务：30分钟自动更新，也支持按需补齐，逐区间隔8秒；初始化未齐时响应体code=202。
|
||||
"""
|
||||
return await get_thirdpart_gold(redis, background_tasks)
|
||||
|
||||
|
||||
@router.get('/gold/history/', operation_id='historyGold', response_model=Response[HistoryGold] | Response[None])
|
||||
@@ -171,7 +122,7 @@ def get_history_gold(region: Annotated[int, Query(..., description='跨区id')],
|
||||
|
||||
|
||||
@router.get('/gold/official/', operation_id='officialGold', response_model=Response[list[OfficialGold]])
|
||||
async def get_official_gold(redis: AsyncRedisDep) -> Response[list[OfficialGold]]:
|
||||
async def get_official_gold(redis: Annotated[AsyncRedis, Depends(thirdpart_gold.get_redis)]) -> Response[list[OfficialGold]]:
|
||||
"""
|
||||
获取DNF官方金币寄售实时数据
|
||||
|
||||
@@ -180,11 +131,43 @@ async def get_official_gold(redis: AsyncRedisDep) -> Response[list[OfficialGold]
|
||||
Note:
|
||||
- 该接口会返回所有跨区的官方金币寄售数据，包括最低成交价排名前5的价格和最高成交价排名前5的价格
|
||||
- 数据来源于DNF官方提供的接口，可能会有一定的延迟和不稳定性
|
||||
- 30分钟自动刷新，失败保留最多24小时的旧值；认证失败且配置了BARK_URL时发送通知
|
||||
"""
|
||||
key = 'openapi:colg:real_time_gold_officical'
|
||||
try:
|
||||
data = await get_redis_info_async(redis, key, _fetch_official_gold, 30 * 60)
|
||||
data = await official_gold.get_data(redis)
|
||||
except Exception:
|
||||
logger.exception('fetch official gold failed')
|
||||
return response(data=[])
|
||||
return response(code=503, data=[], message='官方金价缓存暂不可用')
|
||||
if not data:
|
||||
return response(code=503, data=[], message='官方金价暂不可用，请稍后重试')
|
||||
return response(data=data)
|
||||
|
||||
|
||||
@router.get('/gold/thirdpart/', operation_id='thirdpartGold', response_model=Response[list[RealTimeGold]])
|
||||
async def get_thirdpart_gold(redis: Annotated[AsyncRedis, Depends(thirdpart_gold.get_redis)], background_tasks: BackgroundTasks) -> Response[list[RealTimeGold]]:
|
||||
"""获取跨1～跨8（含跨3A/3B）的第三方首批报价均值，单位：万金币/元。
|
||||
|
||||
不限制最低总价；各平台首次返回的有效报价汇总、去重、按单价取前200条，
|
||||
对每条报价的1/Price取算术平均，不按库存或份数加权。保留上游原始价格，
|
||||
不额外计算交易手续费。该接口无上游分页参数，前200条是本接口的样本口径。
|
||||
|
||||
每30分钟自动后台刷新，跨区页面最短间隔8秒（每分钟不超过8次），Redis锁避免重复刷新。
|
||||
初始化跨区未齐时响应体code=202并注明进度；9区缓存齐全才返回code=200。
|
||||
刷新中可返回部分已完成跨区，缺失跨区不补0。
|
||||
已有缓存即时返回，失败保留原更新时间，旧值最多保留24小时。
|
||||
该接口在DEBUG模式下也使用缓存，Redis不可用时不直接抓取上游。
|
||||
"""
|
||||
try:
|
||||
data = await thirdpart_gold.read_cache(redis)
|
||||
except Exception:
|
||||
logger.exception('read thirdpart gold cache failed')
|
||||
return response(code=503, data=[], message='第三方金价缓存暂不可用')
|
||||
try:
|
||||
await thirdpart_gold.schedule_refresh(redis, background_tasks, data)
|
||||
except Exception:
|
||||
logger.exception('schedule thirdpart gold refresh failed')
|
||||
return response(code=200 if data else 503, data=data, message='第三方金价刷新暂不可用')
|
||||
if len(data) < len(thirdpart_gold.REGIONS):
|
||||
return response(code=202, data=data,
|
||||
message=f'第三方金价缓存尚未齐全（{len(data)}/{len(thirdpart_gold.REGIONS)}跨区），后台刷新中或等待重试')
|
||||
return response(data=data)
|
||||
|
||||
@@ -1,14 +1,30 @@
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
GOLD_REGION_CODES = {
|
||||
'跨1': 'k1', '跨2': 'k2', '跨3A': 'k3a', '跨3B': 'k3b',
|
||||
'跨4': 'k4', '跨5': 'k5', '跨6': 'k6', '跨7': 'k7', '跨8': 'k8',
|
||||
}
|
||||
|
||||
|
||||
class RealTimeGold(BaseModel):
|
||||
updataTime: str = Field(..., description='更新时间')
|
||||
updataTime: int = Field(..., ge=0, description='更新时间，Unix秒级时间戳')
|
||||
"""更新时间"""
|
||||
region: str = Field(..., description='跨区')
|
||||
"""跨区"""
|
||||
goldPrice: float|str = Field(..., description='金价')
|
||||
code: str = Field(..., description='跨区code：k1、k2、k3a、k3b、k4、k5、k6、k7、k8')
|
||||
goldPrice: float = Field(..., gt=0, allow_inf_nan=False, description='金币比例，1元对应的万金币数量')
|
||||
"""金价"""
|
||||
|
||||
|
||||
class ThirdPartGoldCache(RealTimeGold):
|
||||
"""内部缓存数据，不作为接口返回模型。"""
|
||||
|
||||
goldPrice: float = Field(..., gt=0, allow_inf_nan=False, description='首页有效报价换算为万金币/元后的算术平均值')
|
||||
sampleCount: int = Field(..., ge=1, le=200, description='参与均值计算的报价条数')
|
||||
stale: bool = Field(..., description='数据是否已超过30分钟刷新周期')
|
||||
|
||||
|
||||
class DayGold(BaseModel):
|
||||
date: str = Field(..., description='日期')
|
||||
"""日期"""
|
||||
@@ -25,11 +41,14 @@ class HistoryGold(BaseModel):
|
||||
"""echarts数据"""
|
||||
|
||||
class OfficialGold(BaseModel):
|
||||
updataTime: str = Field(..., description='更新时间')
|
||||
updataTime: str = Field(..., pattern=r'^\d+$', description='更新时间，Unix秒级时间戳字符串')
|
||||
"""更新时间"""
|
||||
region: str = Field(..., description='跨区')
|
||||
"""跨区"""
|
||||
goldDealRank: list[float|str] = Field(..., description='金币寄售最低成交价排名前5的价格')
|
||||
code: str = Field(..., description='跨区code：k1、k2、k3a、k3b、k4、k5、k6、k7')
|
||||
goldDealRank: list[float] = Field(..., max_length=5, description='金币寄售最低成交价，最多5项，每100万金币对应的点券数量')
|
||||
"""金币寄售最低成交价排名前5的价格"""
|
||||
goldTopDealRank: list[float|str] = Field(..., description='金币寄售最高成交价排名前5的价格')
|
||||
goldTopDealRank: list[float] = Field(..., max_length=5, description='金币寄售最高成交价，最多5项，每100万金币对应的点券数量')
|
||||
"""金币寄售最高成交价排名前5的价格"""
|
||||
max: float | None = Field(..., description='100点券可兑换的最大万金币数量，无有效报价时为null')
|
||||
min: float | None = Field(..., description='100点券可兑换的最小万金币数量，无有效报价时为null')
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
"""随应用生命周期运行的金币缓存定时检查，无需外部请求触发。"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from api.utils import official_gold, thirdpart_gold
|
||||
|
||||
logger = logging.getLogger('dcalc.gold_refresh')
|
||||
CHECK_INTERVAL_SECONDS = 60
|
||||
_periodic_tasks: list[asyncio.Task] = []
|
||||
|
||||
|
||||
async def refresh_realtime():
|
||||
async with thirdpart_gold.create_redis() as redis:
|
||||
data = await thirdpart_gold.read_cache(redis)
|
||||
await thirdpart_gold.schedule_refresh(redis, None, data)
|
||||
|
||||
|
||||
async def refresh_official():
|
||||
async with thirdpart_gold.create_redis() as redis:
|
||||
await official_gold.refresh_cache(redis)
|
||||
|
||||
|
||||
async def _periodic(job):
|
||||
while True:
|
||||
try:
|
||||
await job()
|
||||
except Exception:
|
||||
logger.exception('periodic gold refresh check failed job=%s', job.__name__)
|
||||
await asyncio.sleep(CHECK_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
async def start_periodic_refresh():
|
||||
if _periodic_tasks:
|
||||
return
|
||||
for job in (refresh_realtime, refresh_official):
|
||||
_periodic_tasks.append(asyncio.create_task(_periodic(job), name=f'gold-periodic-{job.__name__}'))
|
||||
|
||||
|
||||
async def stop_periodic_refresh():
|
||||
tasks = list(_periodic_tasks)
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
_periodic_tasks.clear()
|
||||
@@ -0,0 +1,201 @@
|
||||
"""官方寄售缓存、认证失败识别及可选BARK通知。"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
import aiohttp
|
||||
|
||||
from api.core.Gzip import gzip_unzip, gzip_zip
|
||||
from api.types.colg import GOLD_REGION_CODES, OfficialGold
|
||||
from api.utils.thirdpart_gold import RELEASE_LOCK
|
||||
from config.main import config
|
||||
|
||||
logger = logging.getLogger('dcalc.official_gold')
|
||||
URL = 'https://dzhu.qq.com/zangyi/activity/app'
|
||||
REGIONS = ('1', '2', '3A', '3B', '4', '5', '6', '7')
|
||||
CACHE_KEY = 'openapi:colg:real_time_gold_official:v2'
|
||||
LOCK_KEY = f'{CACHE_KEY}:lock'
|
||||
COOLDOWN_KEY = f'{CACHE_KEY}:cooldown'
|
||||
FRESH_SECONDS = 30 * 60
|
||||
RETENTION_SECONDS = 24 * 60 * 60
|
||||
ERROR_COOLDOWN_SECONDS = 5 * 60
|
||||
ALERT_RETENTION_SECONDS = 24 * 60 * 60
|
||||
|
||||
|
||||
class TokenExpired(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def normalize_row(item: dict) -> dict:
|
||||
"""原报价为点券/100万金币，换算为每100点券的万金币为10000/报价。"""
|
||||
ranks = {}
|
||||
for name in ('goldDealRank', 'goldTopDealRank'):
|
||||
prices = []
|
||||
for raw in item[name][:5]:
|
||||
try:
|
||||
price = float(raw)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
continue
|
||||
if math.isfinite(price):
|
||||
prices.append(price)
|
||||
ranks[name] = prices
|
||||
ratios = [10000 / price for prices in ranks.values() for price in prices
|
||||
if price > 0 and math.isfinite(10000 / price)]
|
||||
return OfficialGold.model_validate({
|
||||
**item, **ranks,
|
||||
'max': round(max(ratios), 4) if ratios else None,
|
||||
'min': round(min(ratios), 4) if ratios else None,
|
||||
}).model_dump()
|
||||
|
||||
|
||||
def is_auth_failure(payload: dict) -> bool:
|
||||
if any(str(payload.get(key)) == '-30003' for key in ('result', 'returnCode')):
|
||||
return True
|
||||
message = ' '.join(str(payload.get(key, '')) for key in ('returnMsg', 'message', 'msg')).lower()
|
||||
return any(marker in message for marker in (
|
||||
'auth verification failed', 'token expired', 'token invalid', 'invalid token',
|
||||
'token失效', 'token已过期', '登录失效', '登录已过期', '请重新登录',
|
||||
))
|
||||
|
||||
|
||||
async def fetch_gold() -> list[dict]:
|
||||
timeout = aiohttp.ClientTimeout(total=10)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async def fetch_region(region):
|
||||
form = {'r': 'godRank/godDealRank', 'userId': config.HELPER_ID,
|
||||
'token': config.HELPER_TOKEN, 'serverNo': region}
|
||||
async with session.post(URL, data=form) as resp:
|
||||
if resp.status == 401:
|
||||
raise TokenExpired('官方金币接口认证失败')
|
||||
payload = await resp.json(content_type=None)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError('官方金币接口返回格式异常')
|
||||
if is_auth_failure(payload):
|
||||
raise TokenExpired('官方金币接口认证失败')
|
||||
resp.raise_for_status()
|
||||
data = payload['data']
|
||||
return normalize_row({
|
||||
'updataTime': str(int(datetime.now(timezone.utc).timestamp())),
|
||||
'region': f'跨{region}', 'code': GOLD_REGION_CODES[f'跨{region}'],
|
||||
'goldDealRank': [float(item['price']) for item in data['godDealRank'][:5]],
|
||||
'goldTopDealRank': [float(item['price']) for item in data['godTopDealRank'][:5]],
|
||||
})
|
||||
|
||||
results = await asyncio.gather(*(fetch_region(region) for region in REGIONS),
|
||||
return_exceptions=True)
|
||||
# 任一跨区认证失败只向刷新调用方抛出一次；优先于其他渠道的网络异常。
|
||||
for result in results:
|
||||
if isinstance(result, TokenExpired):
|
||||
raise result
|
||||
for result in results:
|
||||
if isinstance(result, BaseException):
|
||||
raise result
|
||||
return results
|
||||
|
||||
|
||||
async def notify_token_expired(redis) -> bool:
|
||||
bark_url = (config.BARK_URL or '').strip()
|
||||
if not bark_url:
|
||||
return False
|
||||
# 不将Token或BARK设备key写入缓存键、通知正文或日志。
|
||||
fingerprint = hashlib.sha256(config.HELPER_TOKEN.encode()).hexdigest()
|
||||
key = f'{CACHE_KEY}:token_alert:{fingerprint}'
|
||||
owner = uuid4().hex
|
||||
try:
|
||||
if not await redis.set(key, owner, nx=True, ex=60):
|
||||
return False
|
||||
except Exception:
|
||||
logger.warning('unable to acquire official token notification lock')
|
||||
return False
|
||||
sent = False
|
||||
try:
|
||||
timeout = aiohttp.ClientTimeout(total=5)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.post(bark_url, json={
|
||||
'title': '助手Token失效',
|
||||
'body': '官方接口认证失败，请检查并更新配置',
|
||||
'group': 'dcalc',
|
||||
}, allow_redirects=False) as resp:
|
||||
resp.raise_for_status()
|
||||
payload = await resp.json(content_type=None)
|
||||
if not isinstance(payload, dict) or payload.get('code') != 200:
|
||||
raise ValueError('BARK拒绝推送')
|
||||
await redis.set(key, 'sent', ex=ALERT_RETENTION_SECONDS)
|
||||
sent = True
|
||||
return True
|
||||
except Exception as exc:
|
||||
# ClientResponseError包含完整请求URL，不输出异常正文或堆栈以免暴露设备key。
|
||||
logger.warning('official token BARK notification failed (%s)', type(exc).__name__)
|
||||
return False
|
||||
finally:
|
||||
if not sent:
|
||||
try:
|
||||
await redis.eval(RELEASE_LOCK, 1, key, owner)
|
||||
except Exception:
|
||||
logger.warning('unable to release official token notification lock')
|
||||
|
||||
|
||||
async def read_cache(redis) -> list[dict]:
|
||||
value = await redis.get(CACHE_KEY)
|
||||
if value is None:
|
||||
return []
|
||||
try:
|
||||
rows = [normalize_row(item)
|
||||
for item in json.loads(gzip_unzip(value))]
|
||||
if [row['region'] for row in rows] != [f'跨{region}' for region in REGIONS]:
|
||||
return []
|
||||
if any(not 0 <= time.time() - int(row['updataTime']) < RETENTION_SECONDS for row in rows):
|
||||
return []
|
||||
return rows
|
||||
except (ValueError, TypeError, KeyError, OSError):
|
||||
logger.warning('invalid official gold cache')
|
||||
return []
|
||||
|
||||
|
||||
def is_fresh(rows: list[dict]) -> bool:
|
||||
return (len(rows) == len(REGIONS)
|
||||
and all(0 <= time.time() - int(row['updataTime']) < FRESH_SECONDS for row in rows))
|
||||
|
||||
|
||||
async def refresh_cache(redis) -> bool:
|
||||
if is_fresh(await read_cache(redis)) or await redis.get(COOLDOWN_KEY):
|
||||
return False
|
||||
owner = uuid4().hex
|
||||
if not await redis.set(LOCK_KEY, owner, nx=True, ex=60):
|
||||
return False
|
||||
try:
|
||||
if is_fresh(await read_cache(redis)) or await redis.get(COOLDOWN_KEY):
|
||||
return False
|
||||
try:
|
||||
data = await fetch_gold()
|
||||
except TokenExpired:
|
||||
logger.warning('official gold authentication failed; check HELPER_TOKEN and HELPER_ID')
|
||||
await notify_token_expired(redis)
|
||||
except Exception:
|
||||
logger.exception('fetch official gold failed')
|
||||
else:
|
||||
if await redis.get(LOCK_KEY) not in (owner, owner.encode()):
|
||||
return False
|
||||
await redis.set(CACHE_KEY, gzip_zip(json.dumps(data)), ex=RETENTION_SECONDS)
|
||||
await redis.delete(COOLDOWN_KEY)
|
||||
return True
|
||||
# 网络异常、认证失败均保留上次完整缓存，不写空列表，不改变原更新时间。
|
||||
await redis.set(COOLDOWN_KEY, '1', ex=ERROR_COOLDOWN_SECONDS)
|
||||
return False
|
||||
finally:
|
||||
await redis.eval(RELEASE_LOCK, 1, LOCK_KEY, owner)
|
||||
|
||||
|
||||
async def get_data(redis) -> list[dict]:
|
||||
cached = await read_cache(redis)
|
||||
if cached:
|
||||
return cached
|
||||
# 冷启动时允许请求补齐；与定时任务共用刷新锁，不会重复请求上游。
|
||||
await refresh_cache(redis)
|
||||
return await read_cache(redis)
|
||||
@@ -0,0 +1,305 @@
|
||||
"""YXDR 首次报价均值；Redis 缓存及跨进程刷新锁，所有 HTTP 使用现有 aiohttp。"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
from html.parser import HTMLParser
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
from statistics import fmean
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
from uuid import uuid4
|
||||
|
||||
import aiohttp
|
||||
from redis.asyncio import Redis
|
||||
from redis.asyncio.retry import Retry
|
||||
from redis.backoff import NoBackoff
|
||||
|
||||
from api.types.colg import GOLD_REGION_CODES, ThirdPartGoldCache
|
||||
from config.main import config
|
||||
|
||||
logger = logging.getLogger('dcalc.thirdpart_gold')
|
||||
BASE_URL = 'https://www.yxdr.com'
|
||||
REGIONS = {
|
||||
'kua1': '跨1', 'kua2': '跨2', 'kua3a': '跨3A', 'kua3b': '跨3B',
|
||||
'kua4': '跨4', 'kua5': '跨5', 'kua6': '跨6', 'kua7': '跨7', 'kua8': '跨8',
|
||||
}
|
||||
HEADERS = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
|
||||
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9',
|
||||
}
|
||||
# 网站公开 DNF 页面已停用/移除的渠道；不请求 VIP 专属渠道。
|
||||
DISABLED_PLATFORMS = {1, 12, 21, 50, 51, 52, 53}
|
||||
PAGE_SIZE = 200
|
||||
FRESH_SECONDS = 30 * 60
|
||||
RETENTION_SECONDS = 24 * 60 * 60
|
||||
# 按每分钟最多8次页面访问处理，理论间隔7.5秒，取8秒留出边界余量。
|
||||
# 渠道在区内并发；上游仍可能限频，遇限频停止本轮并保留已有缓存。
|
||||
REGION_INTERVAL = 8
|
||||
LOCK_SECONDS = 15 * 60
|
||||
ERROR_COOLDOWN_SECONDS = 5 * 60
|
||||
REDIS_TIMEOUT_SECONDS = 1
|
||||
CACHE_PREFIX = 'openapi:colg:thirdpart_gold:v1'
|
||||
LOCK_KEY = f'{CACHE_PREFIX}:lock'
|
||||
COOLDOWN_KEY = f'{CACHE_PREFIX}:cooldown'
|
||||
RELEASE_LOCK = """
|
||||
if redis.call('GET', KEYS[1]) == ARGV[1] then
|
||||
return redis.call('DEL', KEYS[1])
|
||||
end
|
||||
return 0
|
||||
"""
|
||||
_refresh_tasks: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
def create_redis():
|
||||
# 只约束本接口；共享池没有显式超时，不能让缓存故障拖住 HTTP 请求。
|
||||
# from_url 创建的连接池由客户端持有，aclose 时一并释放。
|
||||
return Redis.from_url(config.REDIS_URL, protocol=2,
|
||||
socket_connect_timeout=REDIS_TIMEOUT_SECONDS,
|
||||
socket_timeout=REDIS_TIMEOUT_SECONDS,
|
||||
retry=Retry(NoBackoff(), 0))
|
||||
|
||||
|
||||
async def get_redis():
|
||||
async with create_redis() as redis:
|
||||
yield redis
|
||||
|
||||
|
||||
def _refresh_done(task):
|
||||
_refresh_tasks.discard(task)
|
||||
if not task.cancelled():
|
||||
error = task.exception()
|
||||
if error:
|
||||
logger.error('thirdpart gold background task failed',
|
||||
exc_info=(type(error), error, error.__traceback__))
|
||||
|
||||
|
||||
async def launch_refresh(token: str):
|
||||
# BackgroundTasks 只负责启动，HTTP/ASGI 请求不会跟随整轮刷新持续数分钟。
|
||||
task = asyncio.create_task(refresh_cache(token), name='thirdpart-gold-refresh')
|
||||
_refresh_tasks.add(task)
|
||||
task.add_done_callback(_refresh_done)
|
||||
|
||||
|
||||
async def close_refresh_tasks():
|
||||
tasks = list(_refresh_tasks)
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
|
||||
class UpstreamLimited(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class ConfigScripts(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.sources = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
src = dict(attrs).get('src')
|
||||
if tag == 'script' and src and 'CoinSale_' in src:
|
||||
self.sources.append(src)
|
||||
|
||||
|
||||
def read_variable(source: str, name: str):
|
||||
match = re.search(r'\b(?:window\.)?' + re.escape(name) + r'\s*=\s*', source)
|
||||
if not match:
|
||||
raise ValueError(f'YXDR 未下发 {name}')
|
||||
return json.JSONDecoder().raw_decode(source, match.end())[0]
|
||||
|
||||
|
||||
def make_payload(channel: dict, seed: dict, user: dict) -> dict:
|
||||
platform_id = json.loads(channel['data'])['PfId']
|
||||
value = f"{seed['secret']}_{platform_id}_{channel['sign']}"
|
||||
return {
|
||||
'data': channel['data'], 'sign': channel['sign'],
|
||||
'price1': 0, 'price2': 0, 'cross': 0, 'dnfCross': user['dnfCross'],
|
||||
'time': seed['time'], 'secret': hashlib.md5(value.encode('utf-8')).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def first_page_average(offers: list[dict]) -> tuple[float, int]:
|
||||
"""等权平均每条报价的 1/Price（万金币/元），不是 1/平均单价。"""
|
||||
valid = []
|
||||
for offer in offers:
|
||||
try:
|
||||
price = float(offer['Price'])
|
||||
money = float(offer['Money'])
|
||||
number = float(offer['Number'])
|
||||
except (KeyError, TypeError, ValueError, OverflowError):
|
||||
continue
|
||||
if not all(math.isfinite(value) and value > 0 for value in (price, money, number)):
|
||||
continue
|
||||
if not math.isfinite(1 / price):
|
||||
continue
|
||||
valid.append({**offer, 'Price': price})
|
||||
valid.sort(key=lambda item: (item['Price'], item['PfId'], item.get('BuyUrl') or ''))
|
||||
seen, page = set(), []
|
||||
for offer in valid:
|
||||
# 与页面一致，仅对这些平台合并重复商品；7881 同 URL 可有不同规格。
|
||||
key = (offer['PfId'], offer.get('BuyUrl'))
|
||||
if offer['PfId'] in {1, 40, 41} and key[1]:
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
page.append(offer)
|
||||
if len(page) == PAGE_SIZE:
|
||||
break
|
||||
if not page:
|
||||
raise ValueError('YXDR 未返回有效报价')
|
||||
return round(fmean(1 / item['Price'] for item in page), 4), len(page)
|
||||
|
||||
|
||||
async def _read_response(resp) -> str:
|
||||
text = await resp.text()
|
||||
if resp.status == 429 or 'LimitDeny' in text or '访问频率过高' in text:
|
||||
raise UpstreamLimited('YXDR 访问频率受限')
|
||||
resp.raise_for_status()
|
||||
return text
|
||||
|
||||
|
||||
async def fetch_region(session: aiohttp.ClientSession, region: str) -> dict:
|
||||
page_url = f'{BASE_URL}/bijiaqi/dnf/youxibi/{region}'
|
||||
async with session.get(page_url) as resp:
|
||||
page = await _read_response(resp)
|
||||
seed = read_variable(page, 'bijiaSecret')
|
||||
user = read_variable(page, 'bijiaUserInfo')
|
||||
scripts = ConfigScripts()
|
||||
scripts.feed(page)
|
||||
if len(scripts.sources) != 1:
|
||||
raise ValueError('YXDR 区服配置脚本异常')
|
||||
config_url = urljoin(page_url, scripts.sources[0])
|
||||
parts = urlsplit(config_url)
|
||||
if parts.scheme != 'https' or parts.netloc != 'www.yxdr.com':
|
||||
raise ValueError('YXDR 区服配置脚本地址异常')
|
||||
async with session.get(config_url, headers={'Referer': page_url}) as resp:
|
||||
config = await _read_response(resp)
|
||||
channels = [channel for channel in read_variable(config, 'bijiaqiChanels')
|
||||
if json.loads(channel['data'])['PfId'] not in DISABLED_PLATFORMS]
|
||||
if not channels:
|
||||
raise ValueError('YXDR 无可用公开渠道')
|
||||
|
||||
async def fetch_channel(channel):
|
||||
platform = json.loads(channel['data'])
|
||||
if platform['CoinUnit'] != '万金币':
|
||||
raise ValueError('YXDR 金币单位发生变化')
|
||||
headers = {'Referer': page_url, 'Origin': BASE_URL,
|
||||
'X-Requested-With': 'XMLHttpRequest', 'Accept': 'application/json'}
|
||||
async with session.post(f'{BASE_URL}/bijia/coinsale',
|
||||
json=make_payload(channel, seed, user), headers=headers) as resp:
|
||||
result = json.loads(await _read_response(resp))
|
||||
if not result.get('success') or not isinstance(result.get('data'), list):
|
||||
raise ValueError(f"YXDR 渠道 {platform['PfId']}/{platform['CoinNo']} 抓取失败")
|
||||
return [{**offer, 'PfId': platform['PfId']} for offer in result['data']]
|
||||
|
||||
# 同区并发，避免短时签名在顺序请求中失效；任何渠道失败都不覆盖旧均值。
|
||||
results = await asyncio.gather(*(fetch_channel(channel) for channel in channels),
|
||||
return_exceptions=True)
|
||||
# 优先传播限频，即使另一个渠道先失败，也不能继续抓下一个区。
|
||||
for result in results:
|
||||
if isinstance(result, UpstreamLimited):
|
||||
raise result
|
||||
for result in results:
|
||||
if isinstance(result, BaseException):
|
||||
raise result
|
||||
price, count = first_page_average([offer for result in results for offer in result])
|
||||
now = datetime.now(timezone(timedelta(hours=8)))
|
||||
return {'region': REGIONS[region], 'code': GOLD_REGION_CODES[REGIONS[region]],
|
||||
'goldPrice': price, 'sampleCount': count,
|
||||
'updataTime': int(now.timestamp()), 'fetchedAt': now.timestamp()}
|
||||
|
||||
|
||||
async def read_cache(redis) -> list[dict]:
|
||||
values = await redis.mget([f'{CACHE_PREFIX}:{region}' for region in REGIONS])
|
||||
result = []
|
||||
for region, value in zip(REGIONS, values):
|
||||
if value is None:
|
||||
continue
|
||||
try:
|
||||
item = json.loads(value)
|
||||
age = time.time() - item['fetchedAt']
|
||||
if item['region'] != REGIONS[region] or not 0 <= age < RETENTION_SECONDS:
|
||||
continue
|
||||
# 兼容此前缓存的格式化日期及缺少code的记录，无须重新抓取所有跨区。
|
||||
result.append(ThirdPartGoldCache.model_validate({
|
||||
**item, 'updataTime': int(item['fetchedAt']),
|
||||
'code': GOLD_REGION_CODES[REGIONS[region]], 'stale': age >= FRESH_SECONDS,
|
||||
}).model_dump())
|
||||
except (KeyError, TypeError, ValueError):
|
||||
logger.warning('invalid thirdpart gold cache region=%s', region)
|
||||
return result
|
||||
|
||||
|
||||
async def schedule_refresh(redis, background_tasks, data: list[dict]) -> bool:
|
||||
# 缓存均新鲜时不访问上游；失败缺区时遵守全局冷却期。
|
||||
if len(data) == len(REGIONS) and not any(item['stale'] for item in data):
|
||||
return False
|
||||
if await redis.get(COOLDOWN_KEY):
|
||||
return False
|
||||
token = uuid4().hex
|
||||
if not await redis.set(LOCK_KEY, token, nx=True, ex=LOCK_SECONDS):
|
||||
return False
|
||||
try:
|
||||
if await redis.get(COOLDOWN_KEY):
|
||||
await redis.eval(RELEASE_LOCK, 1, LOCK_KEY, token)
|
||||
return False
|
||||
if background_tasks is None:
|
||||
await launch_refresh(token)
|
||||
else:
|
||||
background_tasks.add_task(launch_refresh, token)
|
||||
except BaseException:
|
||||
await redis.eval(RELEASE_LOCK, 1, LOCK_KEY, token)
|
||||
raise
|
||||
return True
|
||||
|
||||
|
||||
async def refresh_cache(token: str):
|
||||
# BackgroundTasks 中不用请求依赖的 Redis 客户端，它可能已被依赖清理器关闭。
|
||||
async with create_redis() as redis:
|
||||
try:
|
||||
if await redis.get(LOCK_KEY) not in (token, token.encode()):
|
||||
return
|
||||
cached = await read_cache(redis)
|
||||
fresh_regions = {item['region'] for item in cached if not item['stale']}
|
||||
pending = [region for region, name in REGIONS.items() if name not in fresh_regions]
|
||||
successful, last_start = 0, None
|
||||
timeout = aiohttp.ClientTimeout(total=15)
|
||||
async with aiohttp.ClientSession(headers=HEADERS, timeout=timeout,
|
||||
connector=aiohttp.TCPConnector(limit=5)) as session:
|
||||
for region in pending:
|
||||
if last_start is not None:
|
||||
await asyncio.sleep(max(0, REGION_INTERVAL - (time.monotonic() - last_start)))
|
||||
# 不在租约到期/已被其他刷新取代后继续请求或写缓存。
|
||||
if await redis.get(LOCK_KEY) not in (token, token.encode()):
|
||||
return
|
||||
last_start = time.monotonic()
|
||||
try:
|
||||
item = await fetch_region(session, region)
|
||||
if await redis.get(LOCK_KEY) not in (token, token.encode()):
|
||||
return
|
||||
await redis.set(f'{CACHE_PREFIX}:{region}', json.dumps(item),
|
||||
ex=RETENTION_SECONDS)
|
||||
successful += 1
|
||||
except UpstreamLimited:
|
||||
logger.warning('thirdpart gold rate limited region=%s', region)
|
||||
break
|
||||
except Exception:
|
||||
logger.exception('fetch thirdpart gold failed region=%s', region)
|
||||
# 新鲜数据由每区 fetchedAt 控制30分钟周期；只对失败设置长冷却。
|
||||
# 下次重试只补缺失/过期区，避免每次重抓前8区导致最后一区一直受限。
|
||||
cooldown = 1 if successful == len(pending) else ERROR_COOLDOWN_SECONDS
|
||||
await redis.set(COOLDOWN_KEY, '1', ex=cooldown)
|
||||
except Exception:
|
||||
logger.exception('refresh thirdpart gold cache failed')
|
||||
finally:
|
||||
try:
|
||||
await redis.eval(RELEASE_LOCK, 1, LOCK_KEY, token)
|
||||
except Exception:
|
||||
logger.exception('release thirdpart gold refresh lock failed')
|
||||
Reference in New Issue
Block a user