feat:添加colg对接功能

backend/api/utils/gold_refresh.py

@@ -0,0 +1,45 @@
+"""随应用生命周期运行的金币缓存定时检查，无需外部请求触发。"""
+
+import asyncio
+import logging
+
+from api.utils import official_gold, thirdpart_gold
+
+logger = logging.getLogger('dcalc.gold_refresh')
+CHECK_INTERVAL_SECONDS = 60
+_periodic_tasks: list[asyncio.Task] = []
+
+
+async def refresh_realtime():
+    async with thirdpart_gold.create_redis() as redis:
+        data = await thirdpart_gold.read_cache(redis)
+        await thirdpart_gold.schedule_refresh(redis, None, data)
+
+
+async def refresh_official():
+    async with thirdpart_gold.create_redis() as redis:
+        await official_gold.refresh_cache(redis)
+
+
+async def _periodic(job):
+    while True:
+        try:
+            await job()
+        except Exception:
+            logger.exception('periodic gold refresh check failed job=%s', job.__name__)
+        await asyncio.sleep(CHECK_INTERVAL_SECONDS)
+
+
+async def start_periodic_refresh():
+    if _periodic_tasks:
+        return
+    for job in (refresh_realtime, refresh_official):
+        _periodic_tasks.append(asyncio.create_task(_periodic(job), name=f'gold-periodic-{job.__name__}'))
+
+
+async def stop_periodic_refresh():
+    tasks = list(_periodic_tasks)
+    for task in tasks:
+        task.cancel()
+    await asyncio.gather(*tasks, return_exceptions=True)
+    _periodic_tasks.clear()

backend/api/utils/official_gold.py

@@ -0,0 +1,201 @@
+"""官方寄售缓存、认证失败识别及可选BARK通知。"""
+
+import asyncio
+from datetime import datetime, timezone
+import hashlib
+import json
+import logging
+import math
+import time
+from uuid import uuid4
+
+import aiohttp
+
+from api.core.Gzip import gzip_unzip, gzip_zip
+from api.types.colg import GOLD_REGION_CODES, OfficialGold
+from api.utils.thirdpart_gold import RELEASE_LOCK
+from config.main import config
+
+logger = logging.getLogger('dcalc.official_gold')
+URL = 'https://dzhu.qq.com/zangyi/activity/app'
+REGIONS = ('1', '2', '3A', '3B', '4', '5', '6', '7')
+CACHE_KEY = 'openapi:colg:real_time_gold_official:v2'
+LOCK_KEY = f'{CACHE_KEY}:lock'
+COOLDOWN_KEY = f'{CACHE_KEY}:cooldown'
+FRESH_SECONDS = 30 * 60
+RETENTION_SECONDS = 24 * 60 * 60
+ERROR_COOLDOWN_SECONDS = 5 * 60
+ALERT_RETENTION_SECONDS = 24 * 60 * 60
+
+
+class TokenExpired(RuntimeError):
+    pass
+
+
+def normalize_row(item: dict) -> dict:
+    """原报价为点券/100万金币，换算为每100点券的万金币为10000/报价。"""
+    ranks = {}
+    for name in ('goldDealRank', 'goldTopDealRank'):
+        prices = []
+        for raw in item[name][:5]:
+            try:
+                price = float(raw)
+            except (TypeError, ValueError, OverflowError):
+                continue
+            if math.isfinite(price):
+                prices.append(price)
+        ranks[name] = prices
+    ratios = [10000 / price for prices in ranks.values() for price in prices
+              if price > 0 and math.isfinite(10000 / price)]
+    return OfficialGold.model_validate({
+        **item, **ranks,
+        'max': round(max(ratios), 4) if ratios else None,
+        'min': round(min(ratios), 4) if ratios else None,
+    }).model_dump()
+
+
+def is_auth_failure(payload: dict) -> bool:
+    if any(str(payload.get(key)) == '-30003' for key in ('result', 'returnCode')):
+        return True
+    message = ' '.join(str(payload.get(key, '')) for key in ('returnMsg', 'message', 'msg')).lower()
+    return any(marker in message for marker in (
+        'auth verification failed', 'token expired', 'token invalid', 'invalid token',
+        'token失效', 'token已过期', '登录失效', '登录已过期', '请重新登录',
+    ))
+
+
+async def fetch_gold() -> list[dict]:
+    timeout = aiohttp.ClientTimeout(total=10)
+    async with aiohttp.ClientSession(timeout=timeout) as session:
+        async def fetch_region(region):
+            form = {'r': 'godRank/godDealRank', 'userId': config.HELPER_ID,
+                    'token': config.HELPER_TOKEN, 'serverNo': region}
+            async with session.post(URL, data=form) as resp:
+                if resp.status == 401:
+                    raise TokenExpired('官方金币接口认证失败')
+                payload = await resp.json(content_type=None)
+                if not isinstance(payload, dict):
+                    raise ValueError('官方金币接口返回格式异常')
+                if is_auth_failure(payload):
+                    raise TokenExpired('官方金币接口认证失败')
+                resp.raise_for_status()
+            data = payload['data']
+            return normalize_row({
+                'updataTime': str(int(datetime.now(timezone.utc).timestamp())),
+                'region': f'跨{region}', 'code': GOLD_REGION_CODES[f'跨{region}'],
+                'goldDealRank': [float(item['price']) for item in data['godDealRank'][:5]],
+                'goldTopDealRank': [float(item['price']) for item in data['godTopDealRank'][:5]],
+            })
+
+        results = await asyncio.gather(*(fetch_region(region) for region in REGIONS),
+                                       return_exceptions=True)
+    # 任一跨区认证失败只向刷新调用方抛出一次；优先于其他渠道的网络异常。
+    for result in results:
+        if isinstance(result, TokenExpired):
+            raise result
+    for result in results:
+        if isinstance(result, BaseException):
+            raise result
+    return results
+
+
+async def notify_token_expired(redis) -> bool:
+    bark_url = (config.BARK_URL or '').strip()
+    if not bark_url:
+        return False
+    # 不将Token或BARK设备key写入缓存键、通知正文或日志。
+    fingerprint = hashlib.sha256(config.HELPER_TOKEN.encode()).hexdigest()
+    key = f'{CACHE_KEY}:token_alert:{fingerprint}'
+    owner = uuid4().hex
+    try:
+        if not await redis.set(key, owner, nx=True, ex=60):
+            return False
+    except Exception:
+        logger.warning('unable to acquire official token notification lock')
+        return False
+    sent = False
+    try:
+        timeout = aiohttp.ClientTimeout(total=5)
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            async with session.post(bark_url, json={
+                'title': '助手Token失效',
+                'body': '官方接口认证失败，请检查并更新配置',
+                'group': 'dcalc',
+            }, allow_redirects=False) as resp:
+                resp.raise_for_status()
+                payload = await resp.json(content_type=None)
+                if not isinstance(payload, dict) or payload.get('code') != 200:
+                    raise ValueError('BARK拒绝推送')
+        await redis.set(key, 'sent', ex=ALERT_RETENTION_SECONDS)
+        sent = True
+        return True
+    except Exception as exc:
+        # ClientResponseError包含完整请求URL，不输出异常正文或堆栈以免暴露设备key。
+        logger.warning('official token BARK notification failed (%s)', type(exc).__name__)
+        return False
+    finally:
+        if not sent:
+            try:
+                await redis.eval(RELEASE_LOCK, 1, key, owner)
+            except Exception:
+                logger.warning('unable to release official token notification lock')
+
+
+async def read_cache(redis) -> list[dict]:
+    value = await redis.get(CACHE_KEY)
+    if value is None:
+        return []
+    try:
+        rows = [normalize_row(item)
+                for item in json.loads(gzip_unzip(value))]
+        if [row['region'] for row in rows] != [f'跨{region}' for region in REGIONS]:
+            return []
+        if any(not 0 <= time.time() - int(row['updataTime']) < RETENTION_SECONDS for row in rows):
+            return []
+        return rows
+    except (ValueError, TypeError, KeyError, OSError):
+        logger.warning('invalid official gold cache')
+        return []
+
+
+def is_fresh(rows: list[dict]) -> bool:
+    return (len(rows) == len(REGIONS)
+            and all(0 <= time.time() - int(row['updataTime']) < FRESH_SECONDS for row in rows))
+
+
+async def refresh_cache(redis) -> bool:
+    if is_fresh(await read_cache(redis)) or await redis.get(COOLDOWN_KEY):
+        return False
+    owner = uuid4().hex
+    if not await redis.set(LOCK_KEY, owner, nx=True, ex=60):
+        return False
+    try:
+        if is_fresh(await read_cache(redis)) or await redis.get(COOLDOWN_KEY):
+            return False
+        try:
+            data = await fetch_gold()
+        except TokenExpired:
+            logger.warning('official gold authentication failed; check HELPER_TOKEN and HELPER_ID')
+            await notify_token_expired(redis)
+        except Exception:
+            logger.exception('fetch official gold failed')
+        else:
+            if await redis.get(LOCK_KEY) not in (owner, owner.encode()):
+                return False
+            await redis.set(CACHE_KEY, gzip_zip(json.dumps(data)), ex=RETENTION_SECONDS)
+            await redis.delete(COOLDOWN_KEY)
+            return True
+        # 网络异常、认证失败均保留上次完整缓存，不写空列表，不改变原更新时间。
+        await redis.set(COOLDOWN_KEY, '1', ex=ERROR_COOLDOWN_SECONDS)
+        return False
+    finally:
+        await redis.eval(RELEASE_LOCK, 1, LOCK_KEY, owner)
+
+
+async def get_data(redis) -> list[dict]:
+    cached = await read_cache(redis)
+    if cached:
+        return cached
+    # 冷启动时允许请求补齐；与定时任务共用刷新锁，不会重复请求上游。
+    await refresh_cache(redis)
+    return await read_cache(redis)

backend/api/utils/thirdpart_gold.py

@@ -0,0 +1,305 @@
+"""YXDR 首次报价均值；Redis 缓存及跨进程刷新锁，所有 HTTP 使用现有 aiohttp。"""
+
+import asyncio
+import hashlib
+from html.parser import HTMLParser
+import json
+import logging
+import math
+import re
+from statistics import fmean
+import time
+from datetime import datetime, timedelta, timezone
+from urllib.parse import urljoin, urlsplit
+from uuid import uuid4
+
+import aiohttp
+from redis.asyncio import Redis
+from redis.asyncio.retry import Retry
+from redis.backoff import NoBackoff
+
+from api.types.colg import GOLD_REGION_CODES, ThirdPartGoldCache
+from config.main import config
+
+logger = logging.getLogger('dcalc.thirdpart_gold')
+BASE_URL = 'https://www.yxdr.com'
+REGIONS = {
+    'kua1': '跨1', 'kua2': '跨2', 'kua3a': '跨3A', 'kua3b': '跨3B',
+    'kua4': '跨4', 'kua5': '跨5', 'kua6': '跨6', 'kua7': '跨7', 'kua8': '跨8',
+}
+HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
+                  'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'zh-CN,zh;q=0.9',
+}
+# 网站公开 DNF 页面已停用/移除的渠道；不请求 VIP 专属渠道。
+DISABLED_PLATFORMS = {1, 12, 21, 50, 51, 52, 53}
+PAGE_SIZE = 200
+FRESH_SECONDS = 30 * 60
+RETENTION_SECONDS = 24 * 60 * 60
+# 按每分钟最多8次页面访问处理，理论间隔7.5秒，取8秒留出边界余量。
+# 渠道在区内并发；上游仍可能限频，遇限频停止本轮并保留已有缓存。
+REGION_INTERVAL = 8
+LOCK_SECONDS = 15 * 60
+ERROR_COOLDOWN_SECONDS = 5 * 60
+REDIS_TIMEOUT_SECONDS = 1
+CACHE_PREFIX = 'openapi:colg:thirdpart_gold:v1'
+LOCK_KEY = f'{CACHE_PREFIX}:lock'
+COOLDOWN_KEY = f'{CACHE_PREFIX}:cooldown'
+RELEASE_LOCK = """
+if redis.call('GET', KEYS[1]) == ARGV[1] then
+    return redis.call('DEL', KEYS[1])
+end
+return 0
+"""
+_refresh_tasks: set[asyncio.Task] = set()
+
+
+def create_redis():
+    # 只约束本接口；共享池没有显式超时，不能让缓存故障拖住 HTTP 请求。
+    # from_url 创建的连接池由客户端持有，aclose 时一并释放。
+    return Redis.from_url(config.REDIS_URL, protocol=2,
+                          socket_connect_timeout=REDIS_TIMEOUT_SECONDS,
+                          socket_timeout=REDIS_TIMEOUT_SECONDS,
+                          retry=Retry(NoBackoff(), 0))
+
+
+async def get_redis():
+    async with create_redis() as redis:
+        yield redis
+
+
+def _refresh_done(task):
+    _refresh_tasks.discard(task)
+    if not task.cancelled():
+        error = task.exception()
+        if error:
+            logger.error('thirdpart gold background task failed',
+                         exc_info=(type(error), error, error.__traceback__))
+
+
+async def launch_refresh(token: str):
+    # BackgroundTasks 只负责启动，HTTP/ASGI 请求不会跟随整轮刷新持续数分钟。
+    task = asyncio.create_task(refresh_cache(token), name='thirdpart-gold-refresh')
+    _refresh_tasks.add(task)
+    task.add_done_callback(_refresh_done)
+
+
+async def close_refresh_tasks():
+    tasks = list(_refresh_tasks)
+    for task in tasks:
+        task.cancel()
+    await asyncio.gather(*tasks, return_exceptions=True)
+
+
+class UpstreamLimited(RuntimeError):
+    pass
+
+
+class ConfigScripts(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.sources = []
+
+    def handle_starttag(self, tag, attrs):
+        src = dict(attrs).get('src')
+        if tag == 'script' and src and 'CoinSale_' in src:
+            self.sources.append(src)
+
+
+def read_variable(source: str, name: str):
+    match = re.search(r'\b(?:window\.)?' + re.escape(name) + r'\s*=\s*', source)
+    if not match:
+        raise ValueError(f'YXDR 未下发 {name}')
+    return json.JSONDecoder().raw_decode(source, match.end())[0]
+
+
+def make_payload(channel: dict, seed: dict, user: dict) -> dict:
+    platform_id = json.loads(channel['data'])['PfId']
+    value = f"{seed['secret']}_{platform_id}_{channel['sign']}"
+    return {
+        'data': channel['data'], 'sign': channel['sign'],
+        'price1': 0, 'price2': 0, 'cross': 0, 'dnfCross': user['dnfCross'],
+        'time': seed['time'], 'secret': hashlib.md5(value.encode('utf-8')).hexdigest(),
+    }
+
+
+def first_page_average(offers: list[dict]) -> tuple[float, int]:
+    """等权平均每条报价的 1/Price（万金币/元），不是 1/平均单价。"""
+    valid = []
+    for offer in offers:
+        try:
+            price = float(offer['Price'])
+            money = float(offer['Money'])
+            number = float(offer['Number'])
+        except (KeyError, TypeError, ValueError, OverflowError):
+            continue
+        if not all(math.isfinite(value) and value > 0 for value in (price, money, number)):
+            continue
+        if not math.isfinite(1 / price):
+            continue
+        valid.append({**offer, 'Price': price})
+    valid.sort(key=lambda item: (item['Price'], item['PfId'], item.get('BuyUrl') or ''))
+    seen, page = set(), []
+    for offer in valid:
+        # 与页面一致，仅对这些平台合并重复商品；7881 同 URL 可有不同规格。
+        key = (offer['PfId'], offer.get('BuyUrl'))
+        if offer['PfId'] in {1, 40, 41} and key[1]:
+            if key in seen:
+                continue
+            seen.add(key)
+        page.append(offer)
+        if len(page) == PAGE_SIZE:
+            break
+    if not page:
+        raise ValueError('YXDR 未返回有效报价')
+    return round(fmean(1 / item['Price'] for item in page), 4), len(page)
+
+
+async def _read_response(resp) -> str:
+    text = await resp.text()
+    if resp.status == 429 or 'LimitDeny' in text or '访问频率过高' in text:
+        raise UpstreamLimited('YXDR 访问频率受限')
+    resp.raise_for_status()
+    return text
+
+
+async def fetch_region(session: aiohttp.ClientSession, region: str) -> dict:
+    page_url = f'{BASE_URL}/bijiaqi/dnf/youxibi/{region}'
+    async with session.get(page_url) as resp:
+        page = await _read_response(resp)
+    seed = read_variable(page, 'bijiaSecret')
+    user = read_variable(page, 'bijiaUserInfo')
+    scripts = ConfigScripts()
+    scripts.feed(page)
+    if len(scripts.sources) != 1:
+        raise ValueError('YXDR 区服配置脚本异常')
+    config_url = urljoin(page_url, scripts.sources[0])
+    parts = urlsplit(config_url)
+    if parts.scheme != 'https' or parts.netloc != 'www.yxdr.com':
+        raise ValueError('YXDR 区服配置脚本地址异常')
+    async with session.get(config_url, headers={'Referer': page_url}) as resp:
+        config = await _read_response(resp)
+    channels = [channel for channel in read_variable(config, 'bijiaqiChanels')
+                if json.loads(channel['data'])['PfId'] not in DISABLED_PLATFORMS]
+    if not channels:
+        raise ValueError('YXDR 无可用公开渠道')
+
+    async def fetch_channel(channel):
+        platform = json.loads(channel['data'])
+        if platform['CoinUnit'] != '万金币':
+            raise ValueError('YXDR 金币单位发生变化')
+        headers = {'Referer': page_url, 'Origin': BASE_URL,
+                   'X-Requested-With': 'XMLHttpRequest', 'Accept': 'application/json'}
+        async with session.post(f'{BASE_URL}/bijia/coinsale',
+                                json=make_payload(channel, seed, user), headers=headers) as resp:
+            result = json.loads(await _read_response(resp))
+        if not result.get('success') or not isinstance(result.get('data'), list):
+            raise ValueError(f"YXDR 渠道 {platform['PfId']}/{platform['CoinNo']} 抓取失败")
+        return [{**offer, 'PfId': platform['PfId']} for offer in result['data']]
+
+    # 同区并发，避免短时签名在顺序请求中失效；任何渠道失败都不覆盖旧均值。
+    results = await asyncio.gather(*(fetch_channel(channel) for channel in channels),
+                                   return_exceptions=True)
+    # 优先传播限频，即使另一个渠道先失败，也不能继续抓下一个区。
+    for result in results:
+        if isinstance(result, UpstreamLimited):
+            raise result
+    for result in results:
+        if isinstance(result, BaseException):
+            raise result
+    price, count = first_page_average([offer for result in results for offer in result])
+    now = datetime.now(timezone(timedelta(hours=8)))
+    return {'region': REGIONS[region], 'code': GOLD_REGION_CODES[REGIONS[region]],
+            'goldPrice': price, 'sampleCount': count,
+            'updataTime': int(now.timestamp()), 'fetchedAt': now.timestamp()}
+
+
+async def read_cache(redis) -> list[dict]:
+    values = await redis.mget([f'{CACHE_PREFIX}:{region}' for region in REGIONS])
+    result = []
+    for region, value in zip(REGIONS, values):
+        if value is None:
+            continue
+        try:
+            item = json.loads(value)
+            age = time.time() - item['fetchedAt']
+            if item['region'] != REGIONS[region] or not 0 <= age < RETENTION_SECONDS:
+                continue
+            # 兼容此前缓存的格式化日期及缺少code的记录，无须重新抓取所有跨区。
+            result.append(ThirdPartGoldCache.model_validate({
+                **item, 'updataTime': int(item['fetchedAt']),
+                'code': GOLD_REGION_CODES[REGIONS[region]], 'stale': age >= FRESH_SECONDS,
+            }).model_dump())
+        except (KeyError, TypeError, ValueError):
+            logger.warning('invalid thirdpart gold cache region=%s', region)
+    return result
+
+
+async def schedule_refresh(redis, background_tasks, data: list[dict]) -> bool:
+    # 缓存均新鲜时不访问上游；失败缺区时遵守全局冷却期。
+    if len(data) == len(REGIONS) and not any(item['stale'] for item in data):
+        return False
+    if await redis.get(COOLDOWN_KEY):
+        return False
+    token = uuid4().hex
+    if not await redis.set(LOCK_KEY, token, nx=True, ex=LOCK_SECONDS):
+        return False
+    try:
+        if await redis.get(COOLDOWN_KEY):
+            await redis.eval(RELEASE_LOCK, 1, LOCK_KEY, token)
+            return False
+        if background_tasks is None:
+            await launch_refresh(token)
+        else:
+            background_tasks.add_task(launch_refresh, token)
+    except BaseException:
+        await redis.eval(RELEASE_LOCK, 1, LOCK_KEY, token)
+        raise
+    return True
+
+
+async def refresh_cache(token: str):
+    # BackgroundTasks 中不用请求依赖的 Redis 客户端，它可能已被依赖清理器关闭。
+    async with create_redis() as redis:
+        try:
+            if await redis.get(LOCK_KEY) not in (token, token.encode()):
+                return
+            cached = await read_cache(redis)
+            fresh_regions = {item['region'] for item in cached if not item['stale']}
+            pending = [region for region, name in REGIONS.items() if name not in fresh_regions]
+            successful, last_start = 0, None
+            timeout = aiohttp.ClientTimeout(total=15)
+            async with aiohttp.ClientSession(headers=HEADERS, timeout=timeout,
+                                             connector=aiohttp.TCPConnector(limit=5)) as session:
+                for region in pending:
+                    if last_start is not None:
+                        await asyncio.sleep(max(0, REGION_INTERVAL - (time.monotonic() - last_start)))
+                    # 不在租约到期/已被其他刷新取代后继续请求或写缓存。
+                    if await redis.get(LOCK_KEY) not in (token, token.encode()):
+                        return
+                    last_start = time.monotonic()
+                    try:
+                        item = await fetch_region(session, region)
+                        if await redis.get(LOCK_KEY) not in (token, token.encode()):
+                            return
+                        await redis.set(f'{CACHE_PREFIX}:{region}', json.dumps(item),
+                                        ex=RETENTION_SECONDS)
+                        successful += 1
+                    except UpstreamLimited:
+                        logger.warning('thirdpart gold rate limited region=%s', region)
+                        break
+                    except Exception:
+                        logger.exception('fetch thirdpart gold failed region=%s', region)
+            # 新鲜数据由每区 fetchedAt 控制30分钟周期；只对失败设置长冷却。
+            # 下次重试只补缺失/过期区，避免每次重抓前8区导致最后一区一直受限。
+            cooldown = 1 if successful == len(pending) else ERROR_COOLDOWN_SECONDS
+            await redis.set(COOLDOWN_KEY, '1', ex=cooldown)
+        except Exception:
+            logger.exception('refresh thirdpart gold cache failed')
+        finally:
+            try:
+                await redis.eval(RELEASE_LOCK, 1, LOCK_KEY, token)
+            except Exception:
+                logger.exception('release thirdpart gold refresh lock failed')