975 lines
28 KiB
Python
975 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
import json
|
|
import os
|
|
import time
|
|
import uuid as _uuid
|
|
from typing import Any, Dict, Optional, Callable
|
|
import socket
|
|
import random
|
|
import sqlite3
|
|
from urllib.parse import quote
|
|
from urllib.request import Request, urlopen, build_opener, ProxyHandler, HTTPSHandler
|
|
from urllib.error import HTTPError, URLError
|
|
import ssl
|
|
|
|
|
|
BASE_URL = "https://cupid.51job.com"
|
|
SIGN_KEY = os.getenv(
|
|
"JOB_SIGN_KEY",
|
|
"abfc8f9dcf8c3f3d8aa294ac5f2cf2cc7767e5592590f39c3f503271dd68562b",
|
|
)
|
|
FROM_DOMAIN = "51job_weixin_wxapp"
|
|
|
|
API_BASE_URL = os.getenv("API_BASE_URL", "http://127.0.0.1:9999")
|
|
COMPANY_INFO_CACHE: Dict[str, Any] = {}
|
|
SUCCESS_LOG_PATH = os.path.join(os.path.dirname(__file__), "success.txt")
|
|
_SUCCESS_WRITTEN: set = set()
|
|
|
|
|
|
def _timestamp() -> int:
|
|
"""Get current UNIX timestamp.
|
|
|
|
Returns:
|
|
int: Current timestamp in seconds.
|
|
"""
|
|
|
|
return int(time.time())
|
|
|
|
|
|
def _encode_query(params: Optional[Dict[str, object]]) -> str:
|
|
"""Encode query parameters preserving insertion order.
|
|
|
|
Args:
|
|
params (Optional[Dict[str, object]]): Query parameters to encode.
|
|
|
|
Returns:
|
|
str: URL-encoded query string starting with '&' when params exist.
|
|
"""
|
|
|
|
if not params:
|
|
return ""
|
|
pieces = []
|
|
for k, v in params.items():
|
|
key = quote(str(k), safe="")
|
|
if isinstance(v, (list, tuple)):
|
|
for item in v:
|
|
pieces.append(f"{key}={quote(str(item), safe='')}")
|
|
elif v is None:
|
|
pieces.append(f"{key}=")
|
|
else:
|
|
pieces.append(f"{key}={quote(str(v), safe='')}")
|
|
return "&" + "&".join(pieces)
|
|
|
|
|
|
def build_signature(
|
|
method: str,
|
|
path: str,
|
|
query_params: Optional[Dict[str, object]] = None,
|
|
body_json: Optional[str] = None,
|
|
timestamp: Optional[int] = None,
|
|
) -> Dict[str, Any]:
|
|
"""Construct signing string and compute HMAC-SHA256 signature.
|
|
|
|
Args:
|
|
method (str): HTTP method (GET or POST).
|
|
path (str): API path without leading slash.
|
|
query_params (Optional[Dict[str, object]]): Query parameters for GET.
|
|
body_json (Optional[str]): JSON body string for POST.
|
|
timestamp (Optional[int]): Provided timestamp; generates if None.
|
|
|
|
Returns:
|
|
Dict[str, Any]: dict with fields 'sig'(hex), 'signed_path', and 'ts'.
|
|
"""
|
|
|
|
import hmac
|
|
import hashlib
|
|
|
|
ts = timestamp or _timestamp()
|
|
base = f"/{path}?api_key=51job×tamp={ts}"
|
|
sign_str = base
|
|
method_u = method.upper()
|
|
if method_u == "GET":
|
|
q = _encode_query(query_params)
|
|
sign_str += q
|
|
signed_path = base + q
|
|
else:
|
|
if body_json:
|
|
sign_str += body_json
|
|
signed_path = base
|
|
key_bytes = SIGN_KEY.encode("utf-8")
|
|
sig = hmac.new(key_bytes, sign_str.encode("utf-8"), hashlib.sha256).hexdigest()
|
|
return {"sig": sig, "signed_path": signed_path, "ts": ts}
|
|
|
|
|
|
def _build_headers(
|
|
sign: str,
|
|
content_type: str,
|
|
uuid: Optional[str] = None,
|
|
account_id: Optional[str] = None,
|
|
user_token: Optional[str] = None,
|
|
partner: Optional[str] = None,
|
|
property_obj: Optional[Dict[str, Any]] = None,
|
|
headers_ext: Optional[Dict[str, str]] = None,
|
|
) -> Dict[str, str]:
|
|
"""Build request headers including signing and context.
|
|
|
|
Args:
|
|
sign (str): Hex-encoded signature.
|
|
content_type (str): Content-Type header.
|
|
uuid (Optional[str]): UUID value for tracing.
|
|
account_id (Optional[str]): Account id.
|
|
user_token (Optional[str]): User token.
|
|
partner (Optional[str]): Partner identifier.
|
|
property_obj (Optional[Dict[str, Any]]): Property payload.
|
|
headers_ext (Optional[Dict[str, str]]): Extra headers to merge.
|
|
|
|
Returns:
|
|
Dict[str, str]: Complete headers dict.
|
|
"""
|
|
|
|
did = uuid or str(_uuid.uuid4())
|
|
headers = {
|
|
"sign": sign,
|
|
"From-Domain": FROM_DOMAIN,
|
|
"Content-Type": content_type,
|
|
"Accept": "application/json",
|
|
"uuid": did,
|
|
}
|
|
if account_id:
|
|
headers["account-id"] = account_id
|
|
if user_token:
|
|
headers["user-token"] = user_token
|
|
if partner:
|
|
headers["partner"] = partner
|
|
|
|
prop = property_obj or {
|
|
"frompageUrl": "",
|
|
"pageUrl": "",
|
|
"isLogin": "是" if bool(account_id) else "否",
|
|
"accountid": account_id or "",
|
|
"resumeId": "",
|
|
"firstFrompageUrl": "",
|
|
"distinct_id": did,
|
|
}
|
|
headers["property"] = quote(json.dumps(prop, ensure_ascii=False), safe="")
|
|
if headers_ext:
|
|
headers.update(headers_ext)
|
|
return headers
|
|
|
|
|
|
def _request(
|
|
method: str,
|
|
path: str,
|
|
params: Optional[Dict[str, Any]] = None,
|
|
body: Optional[Dict[str, Any]] = None,
|
|
uuid: Optional[str] = None,
|
|
account_id: Optional[str] = None,
|
|
user_token: Optional[str] = None,
|
|
partner: Optional[str] = None,
|
|
property_obj: Optional[Dict[str, Any]] = None,
|
|
headers_ext: Optional[Dict[str, str]] = None,
|
|
proxies: Optional[list] = None,
|
|
timeout: int = 10,
|
|
retries: int = 2,
|
|
raw_sink: Optional[Callable[[str], None]] = None,
|
|
) -> Dict[str, Any]:
|
|
"""Execute signed HTTP request with basic retries.
|
|
|
|
Args:
|
|
method (str): HTTP method.
|
|
path (str): API path without leading slash.
|
|
params (Optional[Dict[str, Any]]): Query for GET.
|
|
body (Optional[Dict[str, Any]]): JSON body for POST.
|
|
uuid/account_id/user_token/partner/property_obj: Header context.
|
|
headers_ext (Optional[Dict[str, str]]): Extra headers to merge.
|
|
proxies (Optional[list]): Proxy entries.
|
|
timeout (int): Timeout seconds.
|
|
retries (int): Retry attempts.
|
|
|
|
Returns:
|
|
Dict[str, Any]: Parsed JSON response.
|
|
"""
|
|
|
|
body_json = json.dumps(body, ensure_ascii=False) if body is not None else None
|
|
sig = build_signature(method, path, params, body_json)
|
|
content_type = "application/x-www-form-urlencoded" if method.upper() == "GET" else "application/json"
|
|
headers = _build_headers(
|
|
sign=sig["sig"],
|
|
content_type=content_type,
|
|
uuid=uuid,
|
|
account_id=account_id,
|
|
user_token=user_token,
|
|
partner=partner,
|
|
property_obj=property_obj,
|
|
headers_ext=headers_ext,
|
|
)
|
|
url = f"{BASE_URL}{sig['signed_path']}"
|
|
data_bytes = body_json.encode("utf-8") if (method.upper() == "POST" and body_json is not None) else None
|
|
|
|
attempt = 0
|
|
backoff = 0.5
|
|
last_error: Optional[Exception] = None
|
|
use_insecure_ssl = os.getenv("JOB_INSECURE_SSL") in ("1", "true", "TRUE")
|
|
while attempt <= retries:
|
|
_sleep_between_requests(0.2, 0.7)
|
|
req = Request(url=url, data=data_bytes, headers=headers, method=method.upper())
|
|
try:
|
|
opener = None
|
|
ctx = _get_ssl_context(use_insecure_ssl)
|
|
if proxies:
|
|
idx = attempt % len(proxies)
|
|
p = proxies[idx]
|
|
if isinstance(p, str):
|
|
ph = ProxyHandler({"http": p, "https": p})
|
|
elif isinstance(p, dict):
|
|
ph = ProxyHandler(p)
|
|
else:
|
|
ph = None
|
|
if ph:
|
|
opener = build_opener(ph, HTTPSHandler(context=ctx))
|
|
if opener:
|
|
with opener.open(req, timeout=timeout) as resp:
|
|
payload = resp.read().decode("utf-8")
|
|
else:
|
|
with urlopen(req, timeout=timeout, context=ctx) as resp:
|
|
payload = resp.read().decode("utf-8")
|
|
if raw_sink and isinstance(payload, str):
|
|
try:
|
|
raw_sink(payload)
|
|
except Exception:
|
|
pass
|
|
return json.loads(payload) if payload else {}
|
|
except (HTTPError, URLError) as e:
|
|
print(e)
|
|
last_error = e
|
|
msg = str(e)
|
|
if ("CERTIFICATE_VERIFY_FAILED" in msg) and (not use_insecure_ssl):
|
|
use_insecure_ssl = True
|
|
attempt += 1
|
|
continue
|
|
if attempt == retries:
|
|
return {}
|
|
time.sleep(backoff)
|
|
backoff *= 2
|
|
attempt += 1
|
|
|
|
if last_error:
|
|
return {}
|
|
return {}
|
|
|
|
|
|
def search_company_keyword(
|
|
keyword: str,
|
|
page: int = 1,
|
|
size: int = 20,
|
|
job_area: str = "020000",
|
|
sort_type: str = "0",
|
|
search_type: str = "2",
|
|
scene: str = "12",
|
|
uuid: Optional[str] = None,
|
|
account_id: Optional[str] = None,
|
|
user_token: Optional[str] = None,
|
|
partner: Optional[str] = None,
|
|
property_obj: Optional[Dict[str, Any]] = None,
|
|
raw_sink: Optional[Callable[[str], None]] = None,
|
|
) -> Dict[str, Any]:
|
|
"""Call open/noauth/search with a company keyword.
|
|
|
|
Args:
|
|
keyword (str): Keyword to search.
|
|
page (int): Page number.
|
|
size (int): Page size.
|
|
job_area (str): Area code.
|
|
sort_type (str): Sort type.
|
|
search_type (str): Search type.
|
|
scene (str): Scene id.
|
|
uuid/account_id/user_token/partner/property_obj: Header context.
|
|
|
|
Returns:
|
|
Dict[str, Any]: API response JSON.
|
|
"""
|
|
|
|
params: Dict[str, Any] = {
|
|
"userLonLat": "",
|
|
"sortType": sort_type,
|
|
"keyword": keyword,
|
|
"pageSize": str(size),
|
|
"pageNum": str(page),
|
|
"jobArea": job_area,
|
|
"landmark": "",
|
|
"radius": "",
|
|
"workYear": "",
|
|
"degree": "",
|
|
"companyType": "",
|
|
"companySize": "",
|
|
"salary": "NaN-NaN",
|
|
"jobType": "",
|
|
"metro": "",
|
|
"function": "",
|
|
"industry": "",
|
|
"issueDate": "",
|
|
"searchType": search_type,
|
|
"scene": scene,
|
|
}
|
|
return _request(
|
|
method="GET",
|
|
path="open/noauth/search",
|
|
params=params,
|
|
uuid=uuid,
|
|
account_id=account_id,
|
|
user_token=user_token,
|
|
partner=partner,
|
|
property_obj=property_obj,
|
|
raw_sink=raw_sink,
|
|
)
|
|
|
|
|
|
def company_jobs_by_id(
|
|
co_id: str,
|
|
page: int = 1,
|
|
size: int = 20,
|
|
uuid: Optional[str] = None,
|
|
account_id: Optional[str] = None,
|
|
user_token: Optional[str] = None,
|
|
partner: Optional[str] = None,
|
|
property_obj: Optional[Dict[str, Any]] = None,
|
|
raw_sink: Optional[Callable[[str], None]] = None,
|
|
) -> Dict[str, Any]:
|
|
"""Fetch jobs for a company by numeric id using POST.
|
|
|
|
Args:
|
|
co_id (str): Numeric company id.
|
|
page (int): Page number.
|
|
size (int): Page size.
|
|
uuid/account_id/user_token/partner/property_obj: Header context.
|
|
|
|
Returns:
|
|
Dict[str, Any]: API response JSON.
|
|
"""
|
|
|
|
body: Dict[str, Any] = {
|
|
"pageNum": page,
|
|
"pageSize": size,
|
|
"coId": co_id,
|
|
"scene": 14,
|
|
"requestId": "",
|
|
}
|
|
return _request(
|
|
method="POST",
|
|
path="open/noauth/jobs/company",
|
|
body=body,
|
|
uuid=uuid,
|
|
account_id=account_id,
|
|
user_token=user_token,
|
|
partner=partner,
|
|
property_obj=property_obj,
|
|
raw_sink=raw_sink,
|
|
)
|
|
|
|
|
|
def _extract_items(resp: Dict[str, Any]) -> list:
|
|
"""Extract the first list of items from a nested response.
|
|
|
|
This function searches common keys first, then falls back to a recursive
|
|
traversal to find the first list encountered. It is resilient to schema
|
|
variations of the API response.
|
|
|
|
Args:
|
|
resp (Dict[str, Any]): Parsed JSON response.
|
|
|
|
Returns:
|
|
list: The extracted items list; empty when not found or no data.
|
|
"""
|
|
|
|
if not isinstance(resp, dict):
|
|
return []
|
|
|
|
# Prefer job items under resultbody/job/items
|
|
rb = resp.get("resultbody") or resp.get("resultBody")
|
|
if isinstance(rb, dict):
|
|
job_node = rb.get("job")
|
|
if isinstance(job_node, dict) and isinstance(job_node.get("items"), list):
|
|
return job_node.get("items", [])
|
|
|
|
preferred_keys = (
|
|
"items",
|
|
"list",
|
|
"jobs",
|
|
"jobList",
|
|
"companies",
|
|
"companyList",
|
|
"resultList",
|
|
"dataList",
|
|
)
|
|
|
|
for key in preferred_keys:
|
|
val = resp.get(key)
|
|
if isinstance(val, list):
|
|
return val
|
|
|
|
def _walk(node: Any) -> Optional[list]:
|
|
if isinstance(node, list):
|
|
return node
|
|
if isinstance(node, dict):
|
|
for k in preferred_keys:
|
|
v = node.get(k)
|
|
if isinstance(v, list):
|
|
return v
|
|
for v in node.values():
|
|
found = _walk(v)
|
|
if isinstance(found, list):
|
|
return found
|
|
return None
|
|
|
|
found = _walk(resp)
|
|
return found or []
|
|
|
|
|
|
def _get_local_ip() -> str:
|
|
"""Get local IP address for forwarding header.
|
|
|
|
Returns:
|
|
str: Local IP string.
|
|
"""
|
|
|
|
try:
|
|
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
s.connect(("8.8.8.8", 80))
|
|
ip = s.getsockname()[0]
|
|
s.close()
|
|
return ip
|
|
except Exception:
|
|
return "127.0.0.1"
|
|
|
|
|
|
def _get_ssl_context(insecure: bool = False) -> ssl.SSLContext:
|
|
"""Return SSL context, optionally unverified.
|
|
|
|
Args:
|
|
insecure (bool): Whether to disable certificate verification.
|
|
|
|
Returns:
|
|
ssl.SSLContext: Configured SSL context.
|
|
"""
|
|
|
|
if insecure:
|
|
try:
|
|
return ssl._create_unverified_context()
|
|
except Exception:
|
|
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
|
ctx.check_hostname = False
|
|
ctx.verify_mode = ssl.CERT_NONE
|
|
return ctx
|
|
try:
|
|
return ssl.create_default_context()
|
|
except Exception:
|
|
return ssl._create_unverified_context()
|
|
|
|
def _get_db_path() -> str:
|
|
"""Return default SQLite DB path for storing raw responses.
|
|
|
|
Returns:
|
|
str: Absolute file path to SQLite database.
|
|
"""
|
|
|
|
base_dir = os.path.dirname(__file__)
|
|
return os.path.join(base_dir, "qcwy_raw.sqlite3")
|
|
|
|
|
|
def _init_db(db_path: str) -> None:
|
|
"""Initialize SQLite database with responses table if absent.
|
|
|
|
Args:
|
|
db_path (str): Path to SQLite database file.
|
|
|
|
Returns:
|
|
None
|
|
"""
|
|
|
|
try:
|
|
con = sqlite3.connect(db_path)
|
|
cur = con.cursor()
|
|
cur.execute(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS responses (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
keyword TEXT NOT NULL,
|
|
page INTEGER NOT NULL,
|
|
created_at INTEGER NOT NULL,
|
|
payload TEXT NOT NULL
|
|
)
|
|
"""
|
|
)
|
|
cur.execute(
|
|
"""
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_responses_keyword_page
|
|
ON responses(keyword, page)
|
|
"""
|
|
)
|
|
con.commit()
|
|
con.close()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _save_raw_response(db_path: str, keyword: str, page: int, raw_payload: str) -> None:
|
|
"""Persist raw HTTP response payload into SQLite without modification.
|
|
|
|
Args:
|
|
db_path (str): Path to SQLite database file.
|
|
keyword (str): Search keyword.
|
|
page (int): Page number for the response.
|
|
raw_payload (str): Raw JSON text as received.
|
|
|
|
Returns:
|
|
None
|
|
"""
|
|
|
|
try:
|
|
con = sqlite3.connect(db_path)
|
|
cur = con.cursor()
|
|
cur.execute(
|
|
"INSERT OR IGNORE INTO responses(keyword, page, created_at, payload) VALUES(?, ?, ?, ?)",
|
|
(keyword, int(page), int(time.time()), raw_payload),
|
|
)
|
|
con.commit()
|
|
con.close()
|
|
except Exception:
|
|
pass
|
|
def _has_page_record(db_path: str, keyword: str, page: int) -> bool:
|
|
"""Check if the given keyword+page already exists in SQLite."""
|
|
|
|
try:
|
|
con = sqlite3.connect(db_path)
|
|
cur = con.cursor()
|
|
cur.execute("SELECT 1 FROM responses WHERE keyword=? AND page=? LIMIT 1", (keyword, int(page)))
|
|
row = cur.fetchone()
|
|
con.close()
|
|
return row is not None
|
|
except Exception:
|
|
return False
|
|
|
|
def _make_item_key(it: Any) -> str:
|
|
"""Return a stable deduplication key for an item."""
|
|
|
|
try:
|
|
if isinstance(it, dict):
|
|
for k in ("jobId", "id", "job_id", "positionId"):
|
|
v = it.get(k)
|
|
if v is not None:
|
|
return f"id:{v}"
|
|
return "hash:" + json.dumps(it, ensure_ascii=False, sort_keys=True)
|
|
return "val:" + str(it)
|
|
except Exception:
|
|
return "val:" + str(it)
|
|
def _sleep_between_requests(min_seconds: float = 1.0, max_seconds: float = 3.0) -> None:
|
|
"""Sleep for a random duration between min_seconds and max_seconds.
|
|
|
|
Args:
|
|
min_seconds (float): Minimum seconds to sleep.
|
|
max_seconds (float): Maximum seconds to sleep.
|
|
|
|
Returns:
|
|
None
|
|
"""
|
|
|
|
try:
|
|
dur = random.uniform(min_seconds, max_seconds)
|
|
time.sleep(dur)
|
|
except Exception:
|
|
time.sleep(min_seconds)
|
|
|
|
|
|
def _record_company_success(company_name: Optional[str]) -> None:
|
|
"""Append successful company name to success log file once per process.
|
|
|
|
Args:
|
|
company_name (Optional[str]): Company name string.
|
|
|
|
Returns:
|
|
None
|
|
"""
|
|
|
|
try:
|
|
name = (company_name or "").strip()
|
|
if not name or name in _SUCCESS_WRITTEN:
|
|
return
|
|
with open(SUCCESS_LOG_PATH, "a", encoding="utf-8") as f:
|
|
f.write(f"{name}\n")
|
|
_SUCCESS_WRITTEN.add(name)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _extract_company_name(info: Dict[str, Any]) -> Optional[str]:
|
|
"""Extract company name from company info payload across common keys.
|
|
|
|
Args:
|
|
info (Dict[str, Any]): Company info dict.
|
|
|
|
Returns:
|
|
Optional[str]: Company name when found.
|
|
"""
|
|
|
|
keys = (
|
|
"coname",
|
|
"coName",
|
|
"fullCompanyName",
|
|
"companyName",
|
|
"fullname",
|
|
"name",
|
|
)
|
|
for k in keys:
|
|
v = info.get(k)
|
|
if isinstance(v, str) and v.strip():
|
|
return v.strip()
|
|
for parent in ("basicinfo", "basicInfo"):
|
|
node = info.get(parent)
|
|
if isinstance(node, dict):
|
|
for k in keys:
|
|
v = node.get(k)
|
|
if isinstance(v, str) and v.strip():
|
|
return v.strip()
|
|
return None
|
|
|
|
def _report_universal(items: list, data_type: str = "job") -> bool:
|
|
"""Post items list to universal batch-store-async endpoint.
|
|
|
|
Args:
|
|
items (list): Data list to send.
|
|
data_type (str): Logical data type label.
|
|
|
|
Returns:
|
|
bool: True when accepted, else False.
|
|
"""
|
|
|
|
return False
|
|
|
|
|
|
def _extract_total_count(resp: Dict[str, Any]) -> Optional[int]:
|
|
"""直接从响应中读取 job.totalCount 字段。
|
|
|
|
Args:
|
|
resp (Dict[str, Any]): 解析后的响应 JSON。
|
|
|
|
Returns:
|
|
Optional[int]: 总数，若不存在则为 None。
|
|
"""
|
|
|
|
if not isinstance(resp, dict):
|
|
return None
|
|
|
|
def _direct_get(path: tuple) -> Optional[int]:
|
|
node: Any = resp
|
|
for key in path:
|
|
if not isinstance(node, dict):
|
|
return None
|
|
node = node.get(key)
|
|
if isinstance(node, dict):
|
|
tc = node.get("totalCount")
|
|
if isinstance(tc, int):
|
|
return tc
|
|
if isinstance(tc, str):
|
|
s = tc.strip()
|
|
if s.isdigit():
|
|
return int(s)
|
|
return None
|
|
|
|
for p in (
|
|
("resultbody", "job"),
|
|
("resultBody", "job"),
|
|
("job",),
|
|
("jobs",),
|
|
("result", "job"),
|
|
("data", "job"),
|
|
("payload", "job"),
|
|
):
|
|
v = _direct_get(p)
|
|
if isinstance(v, int):
|
|
return v
|
|
|
|
return None
|
|
|
|
|
|
def paginate_search_company_keyword(
|
|
keyword: str,
|
|
size: int = 20,
|
|
job_area: str = "020000",
|
|
sort_type: str = "0",
|
|
search_type: str = "2",
|
|
scene: str = "12",
|
|
start_page: int = 1,
|
|
max_pages: Optional[int] = None,
|
|
delay: float = 0.2,
|
|
verbose: bool = False,
|
|
db_path: Optional[str] = None,
|
|
) -> list:
|
|
"""Iterate pages for company keyword search until no data.
|
|
|
|
Args:
|
|
keyword (str): Keyword to search.
|
|
size (int): Page size per request.
|
|
job_area (str): Area code.
|
|
sort_type (str): Sort type.
|
|
search_type (str): Search type.
|
|
scene (str): Scene id.
|
|
start_page (int): Starting page number.
|
|
max_pages (Optional[int]): Maximum pages to fetch; None for unlimited.
|
|
delay (float): Delay seconds between requests.
|
|
verbose (bool): Whether to print per-page stats.
|
|
|
|
Returns:
|
|
list: Aggregated items across pages.
|
|
"""
|
|
|
|
results: list = []
|
|
seen_keys: set = set()
|
|
page = start_page
|
|
fetched_pages = 0
|
|
total_count: Optional[int] = None
|
|
|
|
db_path = db_path or _get_db_path()
|
|
_init_db(db_path)
|
|
|
|
while True:
|
|
if max_pages is not None and fetched_pages >= max_pages:
|
|
break
|
|
|
|
if verbose:
|
|
print(json.dumps({"fetching_page": page}, ensure_ascii=False))
|
|
|
|
# Skip crawling when this page is already recorded
|
|
if _has_page_record(db_path, keyword, page):
|
|
if verbose:
|
|
print(json.dumps({"page": page, "skipped": True}, ensure_ascii=False))
|
|
page += 1
|
|
fetched_pages += 1
|
|
ms = delay if delay > 0 else 0.2
|
|
mx = ms * 2
|
|
_sleep_between_requests(ms, mx)
|
|
continue
|
|
|
|
def _sink(raw: str) -> None:
|
|
_save_raw_response(db_path, keyword, page, raw)
|
|
|
|
resp = search_company_keyword(
|
|
keyword=keyword,
|
|
page=page,
|
|
size=size,
|
|
job_area=job_area,
|
|
sort_type=sort_type,
|
|
search_type=search_type,
|
|
scene=scene,
|
|
raw_sink=_sink,
|
|
)
|
|
if verbose:
|
|
print(json.dumps({"page": page, "saved": True}, ensure_ascii=False))
|
|
if total_count is None:
|
|
total_count = _extract_total_count(resp)
|
|
if verbose and total_count is not None:
|
|
print(json.dumps({"totalCount": total_count}, ensure_ascii=False))
|
|
|
|
items = _extract_items(resp)
|
|
filtered: list = []
|
|
for it in items:
|
|
key = _make_item_key(it)
|
|
if key in seen_keys:
|
|
continue
|
|
seen_keys.add(key)
|
|
filtered.append(it)
|
|
if verbose:
|
|
print(json.dumps({"page": page, "items_on_page": len(items), "unique_added": len(filtered)}, ensure_ascii=False))
|
|
|
|
if not filtered:
|
|
break
|
|
results.extend(filtered)
|
|
|
|
if total_count is not None:
|
|
if len(results) >= total_count:
|
|
break
|
|
|
|
page += 1
|
|
fetched_pages += 1
|
|
ms = delay if delay > 0 else 0.2
|
|
mx = ms * 2
|
|
_sleep_between_requests(ms, mx)
|
|
|
|
return results
|
|
|
|
|
|
# Static configuration for pagination demo
|
|
CONFIG: Dict[str, Any] = {
|
|
"keyword": "字节跳动",
|
|
"size": 20,
|
|
"job_area": "020000",
|
|
"sort_type": "0",
|
|
"search_type": "2",
|
|
"scene": "12",
|
|
"start_page": 1,
|
|
"max_pages": None,
|
|
"delay": 0.2,
|
|
"verbose": False,
|
|
"db_path": None,
|
|
}
|
|
|
|
|
|
def main(keyword: str) -> None:
|
|
"""Run a demo of keyword search pagination until no data using static config.
|
|
|
|
Returns:
|
|
None
|
|
"""
|
|
cfg = CONFIG
|
|
|
|
results = paginate_search_company_keyword(
|
|
keyword=keyword,
|
|
size=cfg["size"],
|
|
job_area=cfg["job_area"],
|
|
sort_type=cfg["sort_type"],
|
|
search_type=cfg["search_type"],
|
|
scene=cfg["scene"],
|
|
start_page=cfg["start_page"],
|
|
max_pages=cfg["max_pages"],
|
|
delay=cfg["delay"],
|
|
verbose=cfg["verbose"],
|
|
db_path=cfg["db_path"],
|
|
)
|
|
print(json.dumps({"total_items": len(results)}, ensure_ascii=False))
|
|
|
|
|
|
def get_company_info(company_id: str) -> Dict[str, Any]:
|
|
"""Fetch company details with caching.
|
|
|
|
Args:
|
|
company_id (str): Company identifier string.
|
|
|
|
Returns:
|
|
Dict[str, Any]: Company information dict; empty dict when not found.
|
|
"""
|
|
|
|
if not company_id:
|
|
return {}
|
|
cached = COMPANY_INFO_CACHE.get(company_id)
|
|
if isinstance(cached, dict) and cached:
|
|
return cached
|
|
|
|
params = {
|
|
"companyId": company_id,
|
|
"colorOne": "#ffffff",
|
|
"colorTwo": "#ffffffcc",
|
|
}
|
|
property_obj = {
|
|
"frompageUrl": "",
|
|
"pageUrl": "",
|
|
"isLogin": "否",
|
|
"accountid": "",
|
|
"resumeId": "",
|
|
"firstFrompageUrl": "",
|
|
"distinct_id": str(_uuid.uuid4()),
|
|
"pageCode": "companyDetail|company|companyinfo",
|
|
"shortPageCode": "companyDetail|company|companyinfo",
|
|
}
|
|
|
|
try:
|
|
resp = _request(
|
|
method="GET",
|
|
path="open/noauth/company-info/info-data",
|
|
params=params,
|
|
property_obj=property_obj,
|
|
)
|
|
except Exception:
|
|
resp = {}
|
|
if resp and resp.get("status") in (1, "1"):
|
|
info = resp.get("resultbody", {})
|
|
if isinstance(info, dict) and info:
|
|
COMPANY_INFO_CACHE[company_id] = info
|
|
name = _extract_company_name(info)
|
|
_record_company_success(name)
|
|
return info
|
|
return {}
|
|
return {}
|
|
|
|
|
|
def _enrich_items_with_company_info(resp: Dict[str, Any]) -> list:
|
|
"""Attach company info to items using cache.
|
|
|
|
Args:
|
|
resp (Dict[str, Any]): Parsed JSON response.
|
|
|
|
Returns:
|
|
list: Items with company_info fields when available.
|
|
"""
|
|
items = resp.get("resultbody", {}).get("job", {}).get("items", [])
|
|
enriched = []
|
|
for it in items:
|
|
target = dict(it) if isinstance(it, dict) else {"_value": it}
|
|
co_id = target.get("coId") or target.get("companyId")
|
|
job_id = target.get("jobId")
|
|
city_pinyin = target.get("hrefAreaPinYin")
|
|
if co_id:
|
|
info = get_company_info(str(co_id))
|
|
if info:
|
|
target["company_info"] = info
|
|
target["company_desc"] = (info.get("coinfo", {}) or {}).get("coinfo")
|
|
target["companyHref"] = (info.get("share", {}) or {}).get("weixinshareurl")
|
|
target["jobHref"] = f"https://jobs.51job.com/{city_pinyin}/{job_id}.html"
|
|
nm = _extract_company_name(info) or target.get("fullCompanyName") or target.get("companyName")
|
|
_record_company_success(nm)
|
|
_sleep_between_requests()
|
|
enriched.append(target)
|
|
return enriched
|
|
|
|
|
|
def _load_keywords(path: str) -> list:
|
|
"""Load keywords from a UTF-8 text file, one per line.
|
|
|
|
Args:
|
|
path (str): File path.
|
|
|
|
Returns:
|
|
list: Non-empty trimmed lines.
|
|
"""
|
|
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
lines = [ln.strip() for ln in f.readlines()]
|
|
return [ln for ln in lines if ln]
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def _progress_iter(seq: list, desc: str = "", total: Optional[int] = None):
|
|
"""Iterate with a simple console progress bar.
|
|
|
|
Args:
|
|
seq (list): Items to iterate.
|
|
desc (str): Progress description.
|
|
total (Optional[int]): Total count for percentage.
|
|
|
|
Yields:
|
|
Any: Items from seq.
|
|
"""
|
|
|
|
n = 0
|
|
m = total if total is not None else len(seq)
|
|
bar_len = 24
|
|
for item in seq:
|
|
n += 1
|
|
filled = int(bar_len * n / m) if m else 0
|
|
bar = "#" * filled + "-" * (bar_len - filled)
|
|
pct = int(100 * n / m) if m else 100
|
|
print(f"\r{desc} [{bar}] {n}/{m} {pct}%", end="", flush=True)
|
|
yield item
|
|
print("", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
base_dir = os.path.dirname(__file__)
|
|
fp = os.path.join(base_dir, "company.txt")
|
|
td = _load_keywords(fp) or [CONFIG.get("keyword")]
|
|
for keyword in _progress_iter(td, desc="Keywords", total=len(td)):
|
|
print(keyword)
|
|
main(keyword)
|
|
_sleep_between_requests()
|