feat:添加colg对接功能
This commit is contained in:
@@ -0,0 +1,311 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fetch the initial YXDR DNF coin offers; Python 3 standard library only.
|
||||
|
||||
Examples (run from the repository root):
|
||||
python3 scripts/yxdr_coinsale.py
|
||||
python3 scripts/yxdr_coinsale.py --regions kua1 --price1 100
|
||||
python3 scripts/yxdr_coinsale.py --regions kua3a kua3b --limit 200
|
||||
|
||||
The site has nine region entries: kua1, kua2, kua3a, kua3b, kua4..kua8.
|
||||
For EACH region, fetch fresh bijiaSecret from its HTML, then load the referenced
|
||||
CoinSale_*.js. Each bijiaqiChanels entry supplies an opaque data STRING and sign.
|
||||
POST secret = md5(bijiaSecret.secret + '_' + PfId + '_' + sign), with the
|
||||
unchanged data/sign and the page's time. No page/pageSize parameter is present
|
||||
in the site's coinsale request. We save every initial response and additionally
|
||||
export the cheapest --limit filtered offers (default 200). This cutoff is ours:
|
||||
the UI also has platform-preservation, deduplication and random tie-breaking.
|
||||
|
||||
Do not reuse a captured secret/time across runs or reconstruct the data string:
|
||||
its original representation is signed. Public unauthenticated use only; respect
|
||||
the site's access limits. Default region-start interval is 60 seconds. A limit
|
||||
response stops the entire run, preserving results already written. No retries.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from copy import copy
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from datetime import datetime, timezone
|
||||
from html.parser import HTMLParser
|
||||
from http.cookiejar import CookieJar
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
from urllib.request import HTTPCookieProcessor, Request, build_opener
|
||||
|
||||
|
||||
BASE = "https://www.yxdr.com"
|
||||
REGIONS = {
|
||||
"kua1": "跨一区", "kua2": "跨二区", "kua3a": "跨三A区",
|
||||
"kua3b": "跨三B区", "kua4": "跨四区", "kua5": "跨五区",
|
||||
"kua6": "跨六区", "kua7": "跨七区", "kua8": "跨八区",
|
||||
}
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/131.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
}
|
||||
# Disabled/removed in the public DNF coinsale frontend observed 2026-11-02.
|
||||
DISABLED_PLATFORMS = {1, 12, 21, 50, 51, 52, 53}
|
||||
|
||||
|
||||
class RateLimited(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class Scripts(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.sources = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == "script":
|
||||
src = dict(attrs).get("src")
|
||||
if src:
|
||||
self.sources.append(src)
|
||||
|
||||
|
||||
def read_variable(source, name):
|
||||
"""Decode a JSON assignment without evaluating arbitrary site JavaScript."""
|
||||
match = re.search(r"\b(?:window\.)?" + re.escape(name) + r"\s*=\s*", source)
|
||||
if not match:
|
||||
raise ValueError(f"找不到 {name}；页面可能受限、发生跳转或结构已变化")
|
||||
return json.JSONDecoder().raw_decode(source, match.end())[0]
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self, timeout):
|
||||
self.cookies = CookieJar()
|
||||
self.opener = build_opener(HTTPCookieProcessor(self.cookies))
|
||||
self.timeout = timeout
|
||||
|
||||
def fork(self):
|
||||
client = Client(self.timeout)
|
||||
for cookie in self.cookies:
|
||||
client.cookies.set_cookie(copy(cookie))
|
||||
return client
|
||||
|
||||
def request(self, url, payload=None, referer=None):
|
||||
parts = urlsplit(url)
|
||||
if parts.scheme != "https" or parts.netloc != "www.yxdr.com":
|
||||
raise ValueError(f"拒绝将请求发送到站外地址：{url}")
|
||||
headers = dict(HEADERS)
|
||||
body = None
|
||||
if referer:
|
||||
headers["Referer"] = referer
|
||||
if payload is not None:
|
||||
headers.update({"Content-Type": "application/json; charset=UTF-8",
|
||||
"Accept": "application/json, text/javascript, */*; q=0.01",
|
||||
"Origin": BASE, "X-Requested-With": "XMLHttpRequest"})
|
||||
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
try:
|
||||
with self.opener.open(Request(url, data=body, headers=headers),
|
||||
timeout=self.timeout) as response:
|
||||
source = response.read().decode("utf-8-sig")
|
||||
except HTTPError as exc:
|
||||
source = exc.read().decode("utf-8", errors="replace")
|
||||
if exc.code == 429 or "LimitDeny" in source or "访问频率过高" in source:
|
||||
raise RateLimited("站点限制访问频率，已停止；请稍后再运行") from exc
|
||||
raise RuntimeError(f"HTTP {exc.code}: {url}") from exc
|
||||
if "LimitDeny" in source or "访问频率过高" in source:
|
||||
raise RateLimited("站点限制访问频率，已停止；请稍后再运行")
|
||||
return source
|
||||
|
||||
|
||||
def make_payload(channel, seed, user, price1, price2):
|
||||
platform = json.loads(channel["data"])
|
||||
secret_input = f"{seed['secret']}_{platform['PfId']}_{channel['sign']}"
|
||||
return {
|
||||
"data": channel["data"], # Keep the signed STRING byte-for-byte.
|
||||
"sign": channel["sign"],
|
||||
"price1": price1, "price2": price2,
|
||||
"cross": 0, "dnfCross": user["dnfCross"],
|
||||
"time": seed["time"],
|
||||
"secret": hashlib.md5(secret_input.encode("utf-8")).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def select_offers(rows, price1, price2, limit):
|
||||
selected = []
|
||||
for row in rows:
|
||||
money, number, price = row["Money"], row["Number"], row["Price"]
|
||||
if money != -1 and (money * number < 5 or price == 0):
|
||||
continue
|
||||
if price1 and money * number < price1:
|
||||
continue
|
||||
if price2 and "ling" not in (row.get("TradeType") or "") and money > price2:
|
||||
continue
|
||||
if money >= 1_000_000:
|
||||
continue
|
||||
selected.append(dict(row))
|
||||
selected.sort(key=lambda row: (row["Price"], row["PfId"], row.get("BuyUrl") or ""))
|
||||
# The frontend merges repeated listing URLs for these platforms.
|
||||
result, seen = [], set()
|
||||
for row in selected:
|
||||
key = (row["PfId"], row.get("BuyUrl"))
|
||||
if row["PfId"] in {1, 40, 41} and key[1]:
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
result.append(row)
|
||||
return result[:limit]
|
||||
|
||||
|
||||
def fetch_region(client, region, args):
|
||||
# Get the canonical page; filter values are explicitly sent to coinsale.
|
||||
# The supplied query URL can redirect to the canonical page on a fresh visit.
|
||||
page_url = f"{BASE}/bijiaqi/dnf/youxibi/{region}"
|
||||
page = client.request(page_url)
|
||||
seed = read_variable(page, "bijiaSecret")
|
||||
user = read_variable(page, "bijiaUserInfo")
|
||||
scripts = Scripts()
|
||||
scripts.feed(page)
|
||||
config_src = next((src for src in scripts.sources if "CoinSale_" in src), None)
|
||||
if not config_src:
|
||||
raise ValueError("页面没有下发 CoinSale 区服配置")
|
||||
config_url = urljoin(page_url, config_src)
|
||||
config = client.request(config_url, referer=page_url)
|
||||
channels = read_variable(config, "bijiaqiChanels")
|
||||
report = {
|
||||
"region": region, "region_name": REGIONS[region], "source": page_url,
|
||||
"config_url": config_url, "fetched_at": datetime.now(timezone.utc).isoformat(),
|
||||
"price1": args.price1, "price2": args.price2, "limit": args.limit,
|
||||
"channels": [], "items": [],
|
||||
}
|
||||
rows, requests = [], []
|
||||
for channel in channels:
|
||||
platform = json.loads(channel["data"])
|
||||
pf_id = platform["PfId"]
|
||||
entry = {"platform_id": pf_id, "platform_name": platform["PfName"],
|
||||
"coin_no": platform["CoinNo"], "server": platform["GsName"]}
|
||||
report["channels"].append(entry)
|
||||
if pf_id in DISABLED_PLATFORMS:
|
||||
entry["status"] = "skipped_by_public_frontend"
|
||||
continue
|
||||
payload = make_payload(channel, seed, user, args.price1, args.price2)
|
||||
requests.append((client.fork(), platform, payload, entry))
|
||||
|
||||
def fetch_channel(task):
|
||||
session, platform, payload, entry = task
|
||||
pf_id = platform["PfId"]
|
||||
channel_rows = []
|
||||
try:
|
||||
response = json.loads(session.request(f"{BASE}/bijia/coinsale", payload, page_url))
|
||||
entry["response"] = response # Preserve failed/empty responses, too.
|
||||
if response.get("success") and isinstance(response.get("data"), list):
|
||||
entry["status"] = "success"
|
||||
entry["count"] = len(response["data"])
|
||||
for item in response["data"]:
|
||||
channel_rows.append({**item, "PfId": pf_id, "PfName": platform["PfName"],
|
||||
"CoinNo": platform["CoinNo"], "Region": region,
|
||||
"RegionName": REGIONS[region],
|
||||
"CoinUnit": platform["CoinUnit"],
|
||||
"SourceServer": platform["GsName"]})
|
||||
else:
|
||||
entry["status"] = "failed"
|
||||
entry["error"] = response.get("debug") or "接口返回 success=0 或 data 格式异常"
|
||||
except RateLimited:
|
||||
raise
|
||||
except (RuntimeError, URLError, TimeoutError, ValueError) as exc:
|
||||
entry.update(status="failed", error=str(exc))
|
||||
print(f" {platform['PfName']}/{platform['CoinNo']}: "
|
||||
f"{entry['status']} {entry.get('count', 0)} 条"
|
||||
f" {entry.get('error', '')}", flush=True)
|
||||
return channel_rows
|
||||
|
||||
# The webpage dispatches its channel requests together. Send them while the
|
||||
# freshly issued signing material is current, with an isolated cookie jar
|
||||
# per worker; keep result ordering deterministic.
|
||||
with ThreadPoolExecutor(max_workers=5) as pool:
|
||||
for channel_rows in pool.map(fetch_channel, requests):
|
||||
rows.extend(channel_rows)
|
||||
report["raw_offer_count"] = len(rows)
|
||||
report["items"] = select_offers(rows, args.price1, args.price2, args.limit)
|
||||
active = [entry for entry in report["channels"]
|
||||
if entry["status"] != "skipped_by_public_frontend"]
|
||||
successful = sum(entry["status"] == "success" for entry in active)
|
||||
report["status"] = ("success" if active and successful == len(active)
|
||||
else "partial" if successful else "failed")
|
||||
return report
|
||||
|
||||
|
||||
def save_region(directory, report):
|
||||
name = report["region"]
|
||||
(directory / f"{name}.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
fields = ["Region", "RegionName", "PfId", "PfName", "CoinNo", "SourceServer",
|
||||
"Price", "Amount", "CoinUnit", "Money", "Number", "TradeType", "Seller",
|
||||
"State", "BuyUrl", "ListUrl"]
|
||||
with (directory / f"{name}.csv").open("w", encoding="utf-8-sig", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=fields, extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
writer.writerows(report["items"])
|
||||
|
||||
|
||||
def nonnegative(value):
|
||||
number = float(value)
|
||||
if not math.isfinite(number) or number < 0:
|
||||
raise argparse.ArgumentTypeError("必须是非负有限数值")
|
||||
return number
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--regions", nargs="+", choices=REGIONS, default=list(REGIONS))
|
||||
parser.add_argument("--price1", type=nonnegative, default=0, help="最低总价，元")
|
||||
parser.add_argument("--price2", type=nonnegative, default=0, help="最高总价，0 表示不限")
|
||||
parser.add_argument("--limit", type=int, default=200)
|
||||
parser.add_argument("--interval", type=nonnegative, default=60,
|
||||
help="相邻跨区开始请求的最短间隔秒数，默认60")
|
||||
parser.add_argument("--timeout", type=nonnegative, default=30)
|
||||
parser.add_argument("--output", type=Path,
|
||||
default=Path("tmp/yxdr") / datetime.now().strftime("%Y%m%d-%H%M%S"))
|
||||
args = parser.parse_args()
|
||||
if args.limit < 1 or args.timeout <= 0:
|
||||
parser.error("limit 和 timeout 必须大于0")
|
||||
if args.price2 and args.price1 > args.price2:
|
||||
parser.error("price1 不能大于 price2")
|
||||
args.output.mkdir(parents=True, exist_ok=True)
|
||||
client = Client(args.timeout)
|
||||
summary, last_start, failed = [], None, False
|
||||
for region in dict.fromkeys(args.regions):
|
||||
if last_start is not None:
|
||||
remaining = args.interval - (time.monotonic() - last_start)
|
||||
if remaining > 0:
|
||||
print(f"等待 {remaining:.1f} 秒后获取 {REGIONS[region]}", flush=True)
|
||||
time.sleep(remaining)
|
||||
last_start = time.monotonic()
|
||||
print(f"获取 {REGIONS[region]} ({region})", flush=True)
|
||||
stop = False
|
||||
try:
|
||||
report = fetch_region(client, region, args)
|
||||
save_region(args.output, report)
|
||||
status = report["status"]
|
||||
summary.append({"region": region, "status": status,
|
||||
"raw_count": report["raw_offer_count"],
|
||||
"exported_count": len(report["items"])})
|
||||
failed |= status != "success"
|
||||
except (RateLimited, RuntimeError, URLError, TimeoutError, ValueError) as exc:
|
||||
failed = True
|
||||
summary.append({"region": region, "status": "failed", "error": str(exc)})
|
||||
print(f" 失败：{exc}", file=sys.stderr, flush=True)
|
||||
stop = isinstance(exc, RateLimited)
|
||||
(args.output / "summary.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
if stop:
|
||||
break
|
||||
print(f"输出目录：{args.output.resolve()}", flush=True)
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user