feat:添加colg对接功能

2026-11-03 21:44:30 +08:00
parent 3937185d0a
commit 4709ea4328
13 changed files with 1742 additions and 79 deletions
@@ -0,0 +1,418 @@
+import asyncio
+import hashlib
+import json
+import time
+
+import pytest
+from fastapi import BackgroundTasks
+
+from api.routes import colg
+from api.types.colg import GOLD_REGION_CODES
+from api.utils import thirdpart_gold as gold
+
+
+def offer(price, url='listing', platform=17, money=1):
+    return {'Price': price, 'Money': money, 'Number': 1, 'PfId': platform, 'BuyUrl': url}
+
+
+def cached_item(region='跨1', age=0):
+    fetched_at = time.time() - age
+    return {'region': region, 'goldPrice': 100.0, 'sampleCount': 10,
+            'code': GOLD_REGION_CODES[region], 'updataTime': int(fetched_at),
+            'fetchedAt': fetched_at}
+
+
+class FakeRedis:
+    def __init__(self):
+        self.values = {}
+        self.writes = []
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *args):
+        return False
+
+    async def mget(self, keys):
+        return [self.values.get(key) for key in keys]
+
+    async def get(self, key):
+        return self.values.get(key)
+
+    async def set(self, key, value, nx=False, ex=None):
+        if nx and key in self.values:
+            return False
+        self.values[key] = value
+        self.writes.append((key, value, ex))
+        return True
+
+    async def eval(self, script, number, key, token):
+        if self.values.get(key) == token:
+            del self.values[key]
+            return 1
+        return 0
+
+
+def test_average_is_mean_of_ratios_without_minimum_money_filter():
+    # 100 and 50 万/元 => 75; inverse mean price would incorrectly give 66.67.
+    assert gold.first_page_average([offer(.01), offer(.02)]) == (75.0, 2)
+    invalid = [offer(0), offer(-1), offer(float('nan')), offer(float('inf')),
+               offer(.01, money=-1), offer(None), offer(.01, money=float('nan'))]
+    assert gold.first_page_average(invalid + [offer(.01, money=.5)]) == (100.0, 1)
+    with pytest.raises(ValueError, match='有效报价'):
+        gold.first_page_average(invalid)
+
+
+def test_page_deduplicates_before_cutoff_and_preserves_7881_variants():
+    rows = [offer(.01, 'duplicate', 40), offer(.01, 'duplicate', 40)]
+    rows += [offer(.02, str(i), 17) for i in range(199)]
+    rows += [offer(.5, 'outside', 17)]
+    assert gold.first_page_average(rows) == (50.25, 200)
+    assert gold.first_page_average([offer(.01, platform=44), offer(.02, platform=44)]) == (75.0, 2)
+
+
+def test_request_signature_matches_observed_browser_request():
+    channel = {'data': '{"PfId":17,"CoinNo":"yxb"}',
+               'sign': 'e33b1f23fdbe4677e5e6f8bc7bdcbf78'}
+    seed = {'time': '639292586508765510', 'secret': 'a30ad49b1b2532c403de6319ec4f1c1a'}
+    payload = gold.make_payload(channel, seed, {'dnfCross': 1})
+    assert payload['secret'] == 'b7ac9fd4e4f15735dcbe1411b627bb4e'
+    assert payload['data'] == channel['data']
+    assert payload['time'] == seed['time']
+    assert payload['price1'] == payload['price2'] == 0
+
+
+class FakeResponse:
+    status = 200
+
+    def __init__(self, text):
+        self.body = text
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *args):
+        return False
+
+    async def text(self):
+        return self.body
+
+    def raise_for_status(self):
+        pass
+
+
+def test_region_fetches_its_own_seed_config_and_all_initial_channels():
+    seed = {'time': 'fresh-time', 'secret': 'fresh-seed'}
+    channel = {'data': json.dumps({'PfId': 17, 'CoinNo': 'yxb', 'CoinUnit': '万金币'}),
+               'sign': 'config-sign'}
+    page = ('window.bijiaSecret=' + json.dumps(seed) + ';'
+            'window.bijiaUserInfo={"dnfCross":1};'
+            '<script src="/cate/1838/CoinSale_yxb_254_0.js?ver=935"></script>')
+    posts, gets = [], []
+
+    class Session:
+        def get(self, url, **kwargs):
+            gets.append(url)
+            return FakeResponse(page if '/bijiaqi/' in url else
+                                'window.bijiaqiChanels=' + json.dumps([channel]) + ';')
+
+        def post(self, url, json, **kwargs):
+            posts.append((url, json))
+            return FakeResponse('{"success":1,"data":[{"Price":0.01,"Money":1,"Number":1}]}')
+
+    result = asyncio.run(gold.fetch_region(Session(), 'kua8'))
+    assert result['region'] == '跨8'
+    assert result['code'] == 'k8'
+    assert isinstance(result['updataTime'], int)
+    assert result['goldPrice'] == 100
+    assert result['sampleCount'] == 1
+    assert gets == [gold.BASE_URL + '/bijiaqi/dnf/youxibi/kua8',
+                    gold.BASE_URL + '/cate/1838/CoinSale_yxb_254_0.js?ver=935']
+    assert posts[0][1]['secret'] == hashlib.md5(b'fresh-seed_17_config-sign').hexdigest()
+    assert posts[0][1]['time'] == 'fresh-time'
+    assert 'page' not in posts[0][1]
+
+
+def test_rate_limited_html_does_not_become_empty_quotes():
+    with pytest.raises(gold.UpstreamLimited):
+        asyncio.run(gold._read_response(FakeResponse('<title>LimitDeny</title>')))
+
+
+def test_cache_omits_missing_regions_marks_old_values_and_retains_order():
+    redis = FakeRedis()
+    redis.values[f'{gold.CACHE_PREFIX}:kua1'] = json.dumps(cached_item(age=1900))
+    redis.values[f'{gold.CACHE_PREFIX}:kua2'] = json.dumps(cached_item('跨2', age=90000))
+    redis.values[f'{gold.CACHE_PREFIX}:kua8'] = json.dumps(cached_item('跨8'))
+    rows = asyncio.run(gold.read_cache(redis))
+    assert [row['region'] for row in rows] == ['跨1', '跨8']
+    assert [row['stale'] for row in rows] == [True, False]
+
+
+def test_invalid_cached_average_is_not_served():
+    redis = FakeRedis()
+    redis.values[f'{gold.CACHE_PREFIX}:kua1'] = json.dumps({**cached_item(), 'goldPrice': 0})
+    redis.values[f'{gold.CACHE_PREFIX}:kua8'] = json.dumps({**cached_item('跨8'), 'sampleCount': 0})
+    assert asyncio.run(gold.read_cache(redis)) == []
+
+
+def test_legacy_cache_gets_epoch_and_region_code_without_refetching():
+    redis = FakeRedis()
+    legacy = cached_item('跨3A')
+    legacy.pop('code')
+    legacy['updataTime'] = '2026-11-03 21:13:17'
+    redis.values[f'{gold.CACHE_PREFIX}:kua3a'] = json.dumps(legacy)
+    rows = asyncio.run(gold.read_cache(redis))
+    assert len(rows) == 1
+    assert rows[0]['code'] == 'k3a'
+    assert rows[0]['updataTime'] == int(legacy['fetchedAt'])
+    assert not redis.writes
+
+
+def test_cold_concurrent_requests_schedule_only_one_refresh():
+    redis = FakeRedis()
+    tasks = BackgroundTasks()
+
+    async def run():
+        return await asyncio.gather(*(gold.schedule_refresh(redis, tasks, []) for _ in range(10)))
+
+    assert sum(asyncio.run(run())) == 1
+    assert len(tasks.tasks) == 1
+    assert tasks.tasks[0].func is gold.launch_refresh
+    assert next(ttl for key, _, ttl in redis.writes if key == gold.LOCK_KEY) == gold.LOCK_SECONDS
+
+
+def test_fresh_cache_and_failure_cooldown_do_not_schedule_upstream():
+    redis = FakeRedis()
+    tasks = BackgroundTasks()
+    rows = [{**cached_item(name), 'stale': False} for name in gold.REGIONS.values()]
+    assert asyncio.run(gold.schedule_refresh(redis, tasks, rows)) is False
+    redis.values[gold.COOLDOWN_KEY] = '1'
+    assert asyncio.run(gold.schedule_refresh(redis, tasks, [])) is False
+    assert not tasks.tasks
+    assert gold.LOCK_KEY not in redis.values
+
+
+@pytest.mark.parametrize('limited', [False, True])
+def test_refresh_preserves_failed_region_and_releases_its_lock(monkeypatch, limited):
+    redis = FakeRedis()
+    redis.values[gold.LOCK_KEY] = 'owner'
+    old = json.dumps(cached_item(age=1900))
+    redis.values[f'{gold.CACHE_PREFIX}:kua1'] = old
+    monkeypatch.setattr(gold, 'create_redis', lambda: redis)
+    monkeypatch.setattr(gold, 'REGIONS', {'kua1': '跨1', 'kua8': '跨8'})
+    monkeypatch.setattr(gold, 'REGION_INTERVAL', 0)
+    calls = []
+
+    async def fetch(session, region):
+        calls.append(region)
+        if region == 'kua1':
+            raise gold.UpstreamLimited() if limited else ValueError('channel failed')
+        return cached_item('跨8')
+
+    monkeypatch.setattr(gold, 'fetch_region', fetch)
+    asyncio.run(gold.refresh_cache('owner'))
+    assert redis.values[f'{gold.CACHE_PREFIX}:kua1'] == old
+    assert gold.LOCK_KEY not in redis.values
+    assert redis.values[gold.COOLDOWN_KEY] == '1'
+    if limited:
+        assert calls == ['kua1']
+    else:
+        assert calls == ['kua1', 'kua8']
+        assert f'{gold.CACHE_PREFIX}:kua8' in redis.values
+
+
+def test_refresh_cannot_delete_another_owners_lock(monkeypatch):
+    redis = FakeRedis()
+    redis.values[gold.LOCK_KEY] = 'replacement'
+    monkeypatch.setattr(gold, 'create_redis', lambda: redis)
+    asyncio.run(gold.refresh_cache('expired-owner'))
+    assert redis.values[gold.LOCK_KEY] == 'replacement'
+
+
+def test_route_serializes_existing_contract_and_hides_internal_timestamp(monkeypatch, client):
+    async def read(redis):
+        return [{**cached_item(name), 'stale': False} for name in gold.REGIONS.values()]
+
+    async def schedule(*args):
+        return False
+
+    monkeypatch.setattr(gold, 'read_cache', read)
+    monkeypatch.setattr(gold, 'schedule_refresh', schedule)
+    body = client.get('/api/open/gold/thirdpart/').json()
+    assert body['code'] == 200
+    assert len(body['data']) == 9
+    for row in body['data']:
+        assert set(row) == {'updataTime', 'region', 'code', 'goldPrice'}
+    schema = client.app.openapi()
+    assert schema['paths']['/api/open/gold/thirdpart/']['get']['operationId'] == 'thirdpartGold'
+    assert 'ThirdPartGoldCache' not in schema['components']['schemas']
+
+
+def test_realtime_uses_thirdpart_cache_and_returns_only_defined_fields(monkeypatch, client):
+    async def read(redis):
+        return [{**cached_item(name), 'stale': False} for name in gold.REGIONS.values()]
+
+    schedules = []
+
+    async def schedule(redis, tasks, data):
+        schedules.append(data)
+        return False
+
+    def old_colg_request(*args, **kwargs):
+        raise AssertionError('realtime must no longer request COLG')
+
+    monkeypatch.setattr(gold, 'read_cache', read)
+    monkeypatch.setattr(gold, 'schedule_refresh', schedule)
+    monkeypatch.setattr(colg.requests, 'post', old_colg_request)
+    body = client.get('/api/open/gold/realtime/').json()
+    assert set(body) == {'code', 'message', 'data'}
+    assert body['code'] == 200
+    assert len(body['data']) == 9
+    assert len(schedules) == 1
+    for name, row in zip(gold.REGIONS.values(), body['data']):
+        assert set(row) == {'updataTime', 'region', 'code', 'goldPrice'}
+        assert row['region'] == name
+        assert row['code'] == GOLD_REGION_CODES[name]
+        assert isinstance(row['updataTime'], int)
+        assert isinstance(row['goldPrice'], (int, float))
+
+
+def test_realtime_initialization_still_returns_contract_fields(monkeypatch, client):
+    async def read(redis):
+        return [{**cached_item('跨2'), 'stale': False}]
+
+    async def schedule(*args):
+        return False
+
+    monkeypatch.setattr(gold, 'read_cache', read)
+    monkeypatch.setattr(gold, 'schedule_refresh', schedule)
+    body = client.get('/api/open/gold/realtime/').json()
+    assert body['code'] == 202
+    assert len(body['data']) == 1
+    assert set(body['data'][0]) == {'updataTime', 'region', 'code', 'goldPrice'}
+    assert body['data'][0]['code'] == 'k2'
+
+
+def test_route_cold_start_and_redis_failure_do_not_block_or_scrape(monkeypatch):
+    redis = FakeRedis()
+    tasks = BackgroundTasks()
+    result = asyncio.run(colg.get_thirdpart_gold(redis, tasks))
+    assert result.code == 202 and result.data == []
+    assert len(tasks.tasks) == 1
+
+    async def unavailable(redis):
+        raise ConnectionError('Redis unavailable')
+
+    monkeypatch.setattr(gold, 'read_cache', unavailable)
+    tasks = BackgroundTasks()
+    result = asyncio.run(colg.get_thirdpart_gold(redis, tasks))
+    assert result.code == 503 and result.data == []
+    assert not tasks.tasks
+
+
+def test_route_partial_cache_reports_initialization_progress(monkeypatch):
+    async def read(redis):
+        return [{**cached_item(), 'stale': False}]
+
+    async def schedule(*args):
+        return False
+
+    monkeypatch.setattr(gold, 'read_cache', read)
+    monkeypatch.setattr(gold, 'schedule_refresh', schedule)
+    result = asyncio.run(colg.get_thirdpart_gold(None, BackgroundTasks()))
+    assert result.code == 202
+    assert '1/9' in result.message
+    assert len(result.data) == 1
+
+
+def test_background_launcher_finishes_before_refresh_completes(monkeypatch):
+    async def run():
+        started, finish = asyncio.Event(), asyncio.Event()
+
+        async def refresh(token):
+            started.set()
+            await finish.wait()
+
+        monkeypatch.setattr(gold, 'refresh_cache', refresh)
+        tasks = BackgroundTasks()
+        tasks.add_task(gold.launch_refresh, 'owner')
+        try:
+            await asyncio.wait_for(tasks(), .5)
+            await asyncio.wait_for(started.wait(), .5)
+            assert len(gold._refresh_tasks) == 1
+            assert not next(iter(gold._refresh_tasks)).done()
+        finally:
+            finish.set()
+            await gold.close_refresh_tasks()
+        assert not gold._refresh_tasks
+
+    asyncio.run(run())
+
+
+def test_shutdown_cancels_refresh_and_releases_redis_lock(monkeypatch):
+    redis = FakeRedis()
+    redis.values[gold.LOCK_KEY] = 'owner'
+    monkeypatch.setattr(gold, 'create_redis', lambda: redis)
+
+    async def run():
+        started = asyncio.Event()
+
+        async def fetch(session, region):
+            started.set()
+            await asyncio.Event().wait()
+
+        monkeypatch.setattr(gold, 'fetch_region', fetch)
+        try:
+            await gold.launch_refresh('owner')
+            await asyncio.wait_for(started.wait(), .5)
+        finally:
+            await gold.close_refresh_tasks()
+        assert gold.LOCK_KEY not in redis.values
+        assert not gold._refresh_tasks
+
+    asyncio.run(run())
+
+
+def test_retry_only_fetches_missing_regions_instead_of_repeating_fresh_pages(monkeypatch):
+    redis = FakeRedis()
+    redis.values[gold.LOCK_KEY] = 'owner'
+    for region, name in gold.REGIONS.items():
+        if region != 'kua8':
+            redis.values[f'{gold.CACHE_PREFIX}:{region}'] = json.dumps(cached_item(name))
+    monkeypatch.setattr(gold, 'create_redis', lambda: redis)
+    calls = []
+
+    async def fetch(session, region):
+        calls.append(region)
+        return cached_item(gold.REGIONS[region])
+
+    monkeypatch.setattr(gold, 'fetch_region', fetch)
+    asyncio.run(gold.refresh_cache('owner'))
+    assert calls == ['kua8']
+    assert len(asyncio.run(gold.read_cache(redis))) == 9
+
+
+def test_unresponsive_redis_returns_prompt_error_instead_of_hanging(monkeypatch):
+    async def run():
+        connections = []
+
+        async def blackhole(reader, writer):
+            connections.append(writer)
+            await reader.read()
+
+        server = await asyncio.start_server(blackhole, '127.0.0.1', 0)
+        port = server.sockets[0].getsockname()[1]
+        monkeypatch.setattr(gold.config, 'REDIS_URL', f'redis://127.0.0.1:{port}/0')
+        try:
+            async with gold.create_redis() as redis:
+                result = await asyncio.wait_for(colg.get_thirdpart_gold(redis, BackgroundTasks()), 2.5)
+            assert result.code == 503
+            assert result.data == []
+        finally:
+            server.close()
+            for writer in connections:
+                writer.close()
+                await writer.wait_closed()
+            await server.wait_closed()
+
+    asyncio.run(run())