312 lines
14 KiB
Python
312 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Fetch the initial YXDR DNF coin offers; Python 3 standard library only.
|
|
|
|
Examples (run from the repository root):
|
|
python3 scripts/yxdr_coinsale.py
|
|
python3 scripts/yxdr_coinsale.py --regions kua1 --price1 100
|
|
python3 scripts/yxdr_coinsale.py --regions kua3a kua3b --limit 200
|
|
|
|
The site has nine region entries: kua1, kua2, kua3a, kua3b, kua4..kua8.
|
|
For EACH region, fetch fresh bijiaSecret from its HTML, then load the referenced
|
|
CoinSale_*.js. Each bijiaqiChanels entry supplies an opaque data STRING and sign.
|
|
POST secret = md5(bijiaSecret.secret + '_' + PfId + '_' + sign), with the
|
|
unchanged data/sign and the page's time. No page/pageSize parameter is present
|
|
in the site's coinsale request. We save every initial response and additionally
|
|
export the cheapest --limit filtered offers (default 200). This cutoff is ours:
|
|
the UI also has platform-preservation, deduplication and random tie-breaking.
|
|
|
|
Do not reuse a captured secret/time across runs or reconstruct the data string:
|
|
its original representation is signed. Public unauthenticated use only; respect
|
|
the site's access limits. Default region-start interval is 60 seconds. A limit
|
|
response stops the entire run, preserving results already written. No retries.
|
|
"""
|
|
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from copy import copy
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from datetime import datetime, timezone
|
|
from html.parser import HTMLParser
|
|
from http.cookiejar import CookieJar
|
|
from pathlib import Path
|
|
import re
|
|
import sys
|
|
import time
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urljoin, urlsplit
|
|
from urllib.request import HTTPCookieProcessor, Request, build_opener
|
|
|
|
|
|
BASE = "https://www.yxdr.com"
|
|
REGIONS = {
|
|
"kua1": "跨一区", "kua2": "跨二区", "kua3a": "跨三A区",
|
|
"kua3b": "跨三B区", "kua4": "跨四区", "kua5": "跨五区",
|
|
"kua6": "跨六区", "kua7": "跨七区", "kua8": "跨八区",
|
|
}
|
|
HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/131.0.0.0 Safari/537.36",
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "zh-CN,zh;q=0.9",
|
|
}
|
|
# Disabled/removed in the public DNF coinsale frontend observed 2026-11-02.
|
|
DISABLED_PLATFORMS = {1, 12, 21, 50, 51, 52, 53}
|
|
|
|
|
|
class RateLimited(RuntimeError):
|
|
pass
|
|
|
|
|
|
class Scripts(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.sources = []
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
if tag == "script":
|
|
src = dict(attrs).get("src")
|
|
if src:
|
|
self.sources.append(src)
|
|
|
|
|
|
def read_variable(source, name):
|
|
"""Decode a JSON assignment without evaluating arbitrary site JavaScript."""
|
|
match = re.search(r"\b(?:window\.)?" + re.escape(name) + r"\s*=\s*", source)
|
|
if not match:
|
|
raise ValueError(f"找不到 {name}；页面可能受限、发生跳转或结构已变化")
|
|
return json.JSONDecoder().raw_decode(source, match.end())[0]
|
|
|
|
|
|
class Client:
|
|
def __init__(self, timeout):
|
|
self.cookies = CookieJar()
|
|
self.opener = build_opener(HTTPCookieProcessor(self.cookies))
|
|
self.timeout = timeout
|
|
|
|
def fork(self):
|
|
client = Client(self.timeout)
|
|
for cookie in self.cookies:
|
|
client.cookies.set_cookie(copy(cookie))
|
|
return client
|
|
|
|
def request(self, url, payload=None, referer=None):
|
|
parts = urlsplit(url)
|
|
if parts.scheme != "https" or parts.netloc != "www.yxdr.com":
|
|
raise ValueError(f"拒绝将请求发送到站外地址：{url}")
|
|
headers = dict(HEADERS)
|
|
body = None
|
|
if referer:
|
|
headers["Referer"] = referer
|
|
if payload is not None:
|
|
headers.update({"Content-Type": "application/json; charset=UTF-8",
|
|
"Accept": "application/json, text/javascript, */*; q=0.01",
|
|
"Origin": BASE, "X-Requested-With": "XMLHttpRequest"})
|
|
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
try:
|
|
with self.opener.open(Request(url, data=body, headers=headers),
|
|
timeout=self.timeout) as response:
|
|
source = response.read().decode("utf-8-sig")
|
|
except HTTPError as exc:
|
|
source = exc.read().decode("utf-8", errors="replace")
|
|
if exc.code == 429 or "LimitDeny" in source or "访问频率过高" in source:
|
|
raise RateLimited("站点限制访问频率，已停止；请稍后再运行") from exc
|
|
raise RuntimeError(f"HTTP {exc.code}: {url}") from exc
|
|
if "LimitDeny" in source or "访问频率过高" in source:
|
|
raise RateLimited("站点限制访问频率，已停止；请稍后再运行")
|
|
return source
|
|
|
|
|
|
def make_payload(channel, seed, user, price1, price2):
|
|
platform = json.loads(channel["data"])
|
|
secret_input = f"{seed['secret']}_{platform['PfId']}_{channel['sign']}"
|
|
return {
|
|
"data": channel["data"], # Keep the signed STRING byte-for-byte.
|
|
"sign": channel["sign"],
|
|
"price1": price1, "price2": price2,
|
|
"cross": 0, "dnfCross": user["dnfCross"],
|
|
"time": seed["time"],
|
|
"secret": hashlib.md5(secret_input.encode("utf-8")).hexdigest(),
|
|
}
|
|
|
|
|
|
def select_offers(rows, price1, price2, limit):
|
|
selected = []
|
|
for row in rows:
|
|
money, number, price = row["Money"], row["Number"], row["Price"]
|
|
if money != -1 and (money * number < 5 or price == 0):
|
|
continue
|
|
if price1 and money * number < price1:
|
|
continue
|
|
if price2 and "ling" not in (row.get("TradeType") or "") and money > price2:
|
|
continue
|
|
if money >= 1_000_000:
|
|
continue
|
|
selected.append(dict(row))
|
|
selected.sort(key=lambda row: (row["Price"], row["PfId"], row.get("BuyUrl") or ""))
|
|
# The frontend merges repeated listing URLs for these platforms.
|
|
result, seen = [], set()
|
|
for row in selected:
|
|
key = (row["PfId"], row.get("BuyUrl"))
|
|
if row["PfId"] in {1, 40, 41} and key[1]:
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
result.append(row)
|
|
return result[:limit]
|
|
|
|
|
|
def fetch_region(client, region, args):
|
|
# Get the canonical page; filter values are explicitly sent to coinsale.
|
|
# The supplied query URL can redirect to the canonical page on a fresh visit.
|
|
page_url = f"{BASE}/bijiaqi/dnf/youxibi/{region}"
|
|
page = client.request(page_url)
|
|
seed = read_variable(page, "bijiaSecret")
|
|
user = read_variable(page, "bijiaUserInfo")
|
|
scripts = Scripts()
|
|
scripts.feed(page)
|
|
config_src = next((src for src in scripts.sources if "CoinSale_" in src), None)
|
|
if not config_src:
|
|
raise ValueError("页面没有下发 CoinSale 区服配置")
|
|
config_url = urljoin(page_url, config_src)
|
|
config = client.request(config_url, referer=page_url)
|
|
channels = read_variable(config, "bijiaqiChanels")
|
|
report = {
|
|
"region": region, "region_name": REGIONS[region], "source": page_url,
|
|
"config_url": config_url, "fetched_at": datetime.now(timezone.utc).isoformat(),
|
|
"price1": args.price1, "price2": args.price2, "limit": args.limit,
|
|
"channels": [], "items": [],
|
|
}
|
|
rows, requests = [], []
|
|
for channel in channels:
|
|
platform = json.loads(channel["data"])
|
|
pf_id = platform["PfId"]
|
|
entry = {"platform_id": pf_id, "platform_name": platform["PfName"],
|
|
"coin_no": platform["CoinNo"], "server": platform["GsName"]}
|
|
report["channels"].append(entry)
|
|
if pf_id in DISABLED_PLATFORMS:
|
|
entry["status"] = "skipped_by_public_frontend"
|
|
continue
|
|
payload = make_payload(channel, seed, user, args.price1, args.price2)
|
|
requests.append((client.fork(), platform, payload, entry))
|
|
|
|
def fetch_channel(task):
|
|
session, platform, payload, entry = task
|
|
pf_id = platform["PfId"]
|
|
channel_rows = []
|
|
try:
|
|
response = json.loads(session.request(f"{BASE}/bijia/coinsale", payload, page_url))
|
|
entry["response"] = response # Preserve failed/empty responses, too.
|
|
if response.get("success") and isinstance(response.get("data"), list):
|
|
entry["status"] = "success"
|
|
entry["count"] = len(response["data"])
|
|
for item in response["data"]:
|
|
channel_rows.append({**item, "PfId": pf_id, "PfName": platform["PfName"],
|
|
"CoinNo": platform["CoinNo"], "Region": region,
|
|
"RegionName": REGIONS[region],
|
|
"CoinUnit": platform["CoinUnit"],
|
|
"SourceServer": platform["GsName"]})
|
|
else:
|
|
entry["status"] = "failed"
|
|
entry["error"] = response.get("debug") or "接口返回 success=0 或 data 格式异常"
|
|
except RateLimited:
|
|
raise
|
|
except (RuntimeError, URLError, TimeoutError, ValueError) as exc:
|
|
entry.update(status="failed", error=str(exc))
|
|
print(f" {platform['PfName']}/{platform['CoinNo']}: "
|
|
f"{entry['status']} {entry.get('count', 0)} 条"
|
|
f" {entry.get('error', '')}", flush=True)
|
|
return channel_rows
|
|
|
|
# The webpage dispatches its channel requests together. Send them while the
|
|
# freshly issued signing material is current, with an isolated cookie jar
|
|
# per worker; keep result ordering deterministic.
|
|
with ThreadPoolExecutor(max_workers=5) as pool:
|
|
for channel_rows in pool.map(fetch_channel, requests):
|
|
rows.extend(channel_rows)
|
|
report["raw_offer_count"] = len(rows)
|
|
report["items"] = select_offers(rows, args.price1, args.price2, args.limit)
|
|
active = [entry for entry in report["channels"]
|
|
if entry["status"] != "skipped_by_public_frontend"]
|
|
successful = sum(entry["status"] == "success" for entry in active)
|
|
report["status"] = ("success" if active and successful == len(active)
|
|
else "partial" if successful else "failed")
|
|
return report
|
|
|
|
|
|
def save_region(directory, report):
|
|
name = report["region"]
|
|
(directory / f"{name}.json").write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
fields = ["Region", "RegionName", "PfId", "PfName", "CoinNo", "SourceServer",
|
|
"Price", "Amount", "CoinUnit", "Money", "Number", "TradeType", "Seller",
|
|
"State", "BuyUrl", "ListUrl"]
|
|
with (directory / f"{name}.csv").open("w", encoding="utf-8-sig", newline="") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=fields, extrasaction="ignore")
|
|
writer.writeheader()
|
|
writer.writerows(report["items"])
|
|
|
|
|
|
def nonnegative(value):
|
|
number = float(value)
|
|
if not math.isfinite(number) or number < 0:
|
|
raise argparse.ArgumentTypeError("必须是非负有限数值")
|
|
return number
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
parser.add_argument("--regions", nargs="+", choices=REGIONS, default=list(REGIONS))
|
|
parser.add_argument("--price1", type=nonnegative, default=0, help="最低总价，元")
|
|
parser.add_argument("--price2", type=nonnegative, default=0, help="最高总价，0 表示不限")
|
|
parser.add_argument("--limit", type=int, default=200)
|
|
parser.add_argument("--interval", type=nonnegative, default=60,
|
|
help="相邻跨区开始请求的最短间隔秒数，默认60")
|
|
parser.add_argument("--timeout", type=nonnegative, default=30)
|
|
parser.add_argument("--output", type=Path,
|
|
default=Path("tmp/yxdr") / datetime.now().strftime("%Y%m%d-%H%M%S"))
|
|
args = parser.parse_args()
|
|
if args.limit < 1 or args.timeout <= 0:
|
|
parser.error("limit 和 timeout 必须大于0")
|
|
if args.price2 and args.price1 > args.price2:
|
|
parser.error("price1 不能大于 price2")
|
|
args.output.mkdir(parents=True, exist_ok=True)
|
|
client = Client(args.timeout)
|
|
summary, last_start, failed = [], None, False
|
|
for region in dict.fromkeys(args.regions):
|
|
if last_start is not None:
|
|
remaining = args.interval - (time.monotonic() - last_start)
|
|
if remaining > 0:
|
|
print(f"等待 {remaining:.1f} 秒后获取 {REGIONS[region]}", flush=True)
|
|
time.sleep(remaining)
|
|
last_start = time.monotonic()
|
|
print(f"获取 {REGIONS[region]} ({region})", flush=True)
|
|
stop = False
|
|
try:
|
|
report = fetch_region(client, region, args)
|
|
save_region(args.output, report)
|
|
status = report["status"]
|
|
summary.append({"region": region, "status": status,
|
|
"raw_count": report["raw_offer_count"],
|
|
"exported_count": len(report["items"])})
|
|
failed |= status != "success"
|
|
except (RateLimited, RuntimeError, URLError, TimeoutError, ValueError) as exc:
|
|
failed = True
|
|
summary.append({"region": region, "status": "failed", "error": str(exc)})
|
|
print(f" 失败：{exc}", file=sys.stderr, flush=True)
|
|
stop = isinstance(exc, RateLimited)
|
|
(args.output / "summary.json").write_text(
|
|
json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
if stop:
|
|
break
|
|
print(f"输出目录：{args.output.resolve()}", flush=True)
|
|
return 1 if failed else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|