232 lines
7.5 KiB
Python
232 lines
7.5 KiB
Python
import os
|
||
import time
|
||
import json
|
||
import hmac
|
||
import hashlib
|
||
import random
|
||
from typing import Any, Dict, List, Optional
|
||
from urllib.parse import quote
|
||
|
||
import requests
|
||
import urllib.parse
|
||
import urllib3
|
||
|
||
|
||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||
|
||
|
||
BASE_URL = "https://we.51job.com/api/job/search-pc"
|
||
API_BASE_URL = os.getenv("API_BASE_URL", "http://127.0.0.1:9999")
|
||
SIGN_KEY = "abfc8f9dcf8c3f3d8aa294ac5f2cf2cc7767e5592590f39c3f503271dd68562b"
|
||
|
||
|
||
def _now_ts() -> str:
|
||
"""返回当前时间戳字符串"""
|
||
return str(int(time.time()))
|
||
|
||
|
||
def _build_proxy() -> Dict[str, str]:
|
||
"""构造HTTP/HTTPS代理配置"""
|
||
url = os.getenv("PROXY_URL", "http://t13319619426654:ln8aj9nl@s432.kdltps.com:15818")
|
||
return {"http": url, "https": url}
|
||
|
||
|
||
def _hmac_sign_url(url: str) -> str:
|
||
"""对完整URL进行HMAC-SHA256签名"""
|
||
return hmac.new(SIGN_KEY.encode("utf-8"), url.encode("utf-8"), hashlib.sha256).hexdigest()
|
||
|
||
|
||
def _generate_acw_sc_v2(arg1: str) -> str:
|
||
"""依据arg1生成acw_sc__v2值"""
|
||
if not arg1:
|
||
return ""
|
||
pos_list = [15, 35, 29, 24, 33, 16, 1, 38, 10, 9, 19, 31, 40, 27, 22, 23, 25, 13, 6, 11, 39, 18, 20, 8, 14, 21,
|
||
32, 26, 2, 30, 7, 4, 17, 5, 3, 28, 34, 37, 12, 36]
|
||
mask = "3000176000856006061501533003690027800375"
|
||
out = [None] * len(pos_list)
|
||
for i in range(len(arg1)):
|
||
for j in range(len(pos_list)):
|
||
if pos_list[j] == i + 1:
|
||
out[j] = arg1[i]
|
||
break
|
||
arg2 = ''.join([c for c in out if c])
|
||
arg3 = ""
|
||
for i in range(0, min(len(arg2), len(mask)), 2):
|
||
sc = int(arg2[i:i + 2], 16)
|
||
mc = int(mask[i:i + 2], 16)
|
||
arg3 += format(sc ^ mc, '02x')
|
||
return arg3
|
||
|
||
|
||
def _build_web_headers(keyword: str, did: str, sign: str) -> Dict[str, str]:
|
||
"""构造网页接口所需请求头"""
|
||
encoded_kw = quote(keyword, safe="")
|
||
referer = f"https://we.51job.com/pc/search?jobArea=020000&keyword={encoded_kw}&searchType=2&keywordType="
|
||
web_prop = {
|
||
"partner": "",
|
||
"webId": 2,
|
||
"fromdomain": "51job_web",
|
||
"frompageUrl": "https://we.51job.com/",
|
||
"pageUrl": referer,
|
||
"identityType": "",
|
||
"userType": "",
|
||
"isLogin": "否",
|
||
"accountid": "",
|
||
"keywordType": "直接输入",
|
||
}
|
||
return {
|
||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
|
||
"Accept": "application/json, text/plain, */*",
|
||
"accept-language": "zh-CN,zh;q=0.9",
|
||
"from-domain": "51job_web",
|
||
"priority": "u=1, i",
|
||
"property": quote(json.dumps(web_prop, ensure_ascii=False), safe=""),
|
||
"referer": referer,
|
||
"sec-ch-ua": '"Not/A)Brand";v="8", "Chromium";v="136", "Google Chrome";v="136"',
|
||
"sec-ch-ua-mobile": "?0",
|
||
"sec-ch-ua-platform": '"macOS"',
|
||
"sec-fetch-dest": "empty",
|
||
"sec-fetch-mode": "cors",
|
||
"sec-fetch-site": "same-origin",
|
||
"sign": sign,
|
||
"uuid": did,
|
||
"Cookie": f"guid={did}",
|
||
}
|
||
|
||
|
||
def _build_query(city_code: str, page: int, keyword: str) -> str:
|
||
"""构造search-pc查询串"""
|
||
params = {
|
||
'api_key': '51job',
|
||
'timestamp': _now_ts(),
|
||
'keyword': keyword,
|
||
'searchType': '2',
|
||
'function': '',
|
||
'industry': '',
|
||
'jobArea': city_code,
|
||
'jobArea2': '',
|
||
'landmark': '',
|
||
'metro': '',
|
||
'salary': '',
|
||
'workYear': '',
|
||
'degree': '',
|
||
'companyType': '',
|
||
'companySize': '',
|
||
'jobType': '',
|
||
'issueDate': '',
|
||
'sortType': '0',
|
||
'pageNum': str(page),
|
||
'requestId': '',
|
||
'pageSize': '20',
|
||
'source': '1',
|
||
'accountId': '',
|
||
'pageCode': 'sou|sou|soulb',
|
||
'scene': '7',
|
||
}
|
||
return urllib.parse.urlencode(params, quote_via=urllib.parse.quote)
|
||
|
||
|
||
def _prefetch_acw(city_code: str, page: int, keyword: str, proxies: Dict[str, str]) -> Optional[str]:
|
||
"""预取arg1并生成acw_sc__v2"""
|
||
qs = _build_query(city_code, page, keyword)
|
||
full_url = f"{BASE_URL}?{qs}"
|
||
did = str(random.randint(10**15, 10**16-1))
|
||
sign = _hmac_sign_url(full_url)
|
||
headers = _build_web_headers(keyword, did, sign)
|
||
try:
|
||
resp = requests.get(full_url, headers=headers, timeout=20, proxies=proxies, verify=False)
|
||
txt = resp.text or ""
|
||
m = re_search_arg1(txt)
|
||
if not m:
|
||
return None
|
||
return _generate_acw_sc_v2(m)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def re_search_arg1(text: str) -> Optional[str]:
|
||
"""从文本中提取arg1"""
|
||
import re
|
||
m = re.search(r"var\s+arg1\s*=\s*['\"]([^'\"]+)['\"]", text)
|
||
return m.group(1) if m else None
|
||
|
||
|
||
def fetch_page(city_code: str, keyword: str, page: int, proxies: Dict[str, str]) -> Dict[str, Any]:
|
||
"""抓取单页数据并返回解析后的JSON或文本封装"""
|
||
qs = _build_query(city_code, page, keyword)
|
||
full_url = f"{BASE_URL}?{qs}"
|
||
did = str(random.randint(10**15, 10**16-1))
|
||
sign = _hmac_sign_url(full_url)
|
||
headers = _build_web_headers(keyword, did, sign)
|
||
acw = _prefetch_acw(city_code, page, keyword, proxies)
|
||
if acw:
|
||
headers["Cookie"] = headers.get("Cookie", "") + f"; acw_sc__v2={acw}"
|
||
resp = requests.get(full_url, headers=headers, timeout=30, proxies=proxies, verify=False)
|
||
try:
|
||
return resp.json()
|
||
except ValueError:
|
||
return {"raw": resp.text}
|
||
|
||
|
||
def load_company_keywords() -> List[str]:
|
||
"""读取同目录company.txt为关键词列表"""
|
||
fp = os.path.join(os.path.dirname(__file__), "company.txt")
|
||
try:
|
||
with open(fp, "r", encoding="utf-8") as f:
|
||
lines = [ln.strip() for ln in f.readlines()]
|
||
return [ln for ln in lines if ln]
|
||
except Exception:
|
||
return []
|
||
|
||
|
||
def fetch_service_params() -> Optional[Dict[str, Any]]:
|
||
"""从服务端获取当天未使用的检索条件并占用
|
||
|
||
返回:
|
||
dict: {"city_code": str, "keyword": str} 或 None
|
||
"""
|
||
try:
|
||
url = f"{API_BASE_URL}/api/v1/keyword/available"
|
||
r = requests.get(url, params={"source": "qcwy", "limit": 1}, timeout=10)
|
||
if r.status_code != 200:
|
||
return None
|
||
js = r.json()
|
||
data = js.get("data") or {}
|
||
items = data.get("items") or []
|
||
if not items:
|
||
return None
|
||
item = items[0]
|
||
ids = [item.get("id")]
|
||
if ids and ids[0]:
|
||
try:
|
||
murl = f"{API_BASE_URL}/api/v1/keyword/mark-used"
|
||
requests.post(murl, json={"source": "qcwy", "ids": ids}, timeout=10)
|
||
except Exception:
|
||
pass
|
||
return {"city_code": str(item.get("city", "")), "keyword": str(item.get("job", ""))}
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def run(city_code: str = "020000", max_pages: int = 3) -> None:
|
||
"""执行基于company.txt的固定条件搜索并输出返回数据"""
|
||
proxies = _build_proxy()
|
||
# 优先从服务端拉取城市与关键词；无数据时回退到本地company.txt
|
||
svc = fetch_service_params()
|
||
keywords = [svc["keyword"]] if svc else load_company_keywords()
|
||
city_code = svc["city_code"] if svc else city_code
|
||
for kw in keywords:
|
||
for p in range(1, max_pages + 1):
|
||
data = fetch_page(city_code, kw, p, proxies)
|
||
payload = {
|
||
"keyword": kw,
|
||
"page": p,
|
||
"data": data,
|
||
}
|
||
print(json.dumps(payload, ensure_ascii=False))
|
||
time.sleep(random.uniform(0.8, 1.6))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
run()
|