1298 lines
44 KiB
Python
1298 lines
44 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
51job 职位爬虫脚本
|
||
包含代理IP管理、远程数据上报和签名算法功能
|
||
"""
|
||
|
||
import httpx
|
||
import socket
|
||
import time
|
||
import hashlib
|
||
import hmac
|
||
import json
|
||
import random
|
||
from loguru import logger
|
||
import os
|
||
from datetime import datetime
|
||
from typing import List, Dict, Optional, Any, Tuple
|
||
from dataclasses import dataclass
|
||
from urllib.parse import urljoin, quote
|
||
|
||
import requests
|
||
|
||
os.makedirs("logs", exist_ok=True)
|
||
logger.add("logs/log_{time:YYYY-MM-DD}.log", level="INFO", rotation="00:00", retention="30 days", enqueue=True)
|
||
|
||
|
||
API_BASE_URL = os.getenv('API_BASE_URL', 'http://124.222.106.226:9999')
|
||
|
||
API_PUBLIC_HOST = os.getenv('API_PUBLIC_HOST')
|
||
PROXY_URL = "http://t13319619426654:ln8aj9nl@s432.kdltps.com:15818"
|
||
|
||
|
||
def sleep_random_between() -> float:
|
||
try:
|
||
min_seconds = float(os.getenv('SLEEP_MIN_SECONDS', '1'))
|
||
max_seconds = float(os.getenv('SLEEP_MAX_SECONDS', '10'))
|
||
if max_seconds < min_seconds:
|
||
max_seconds = min_seconds
|
||
wait_time = random.uniform(min_seconds, max_seconds)
|
||
except Exception:
|
||
wait_time = 1.0
|
||
time.sleep(wait_time)
|
||
return wait_time
|
||
|
||
def get_job_id(job: Dict[str, Any]) -> Optional[str]:
|
||
"""从职位数据中提取jobId作为唯一标识"""
|
||
return job.get('jobId') or job.get('id') or job.get('job_id') or None
|
||
|
||
def deduplicate_jobs(jobs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||
"""对职位列表进行去重，基于jobId"""
|
||
seen_ids = set()
|
||
unique_jobs = []
|
||
for job in jobs:
|
||
job_id = get_job_id(job)
|
||
if job_id:
|
||
if job_id not in seen_ids:
|
||
seen_ids.add(job_id)
|
||
unique_jobs.append(job)
|
||
else:
|
||
# 如果没有jobId，保留该条记录（可能是异常数据）
|
||
unique_jobs.append(job)
|
||
return unique_jobs
|
||
|
||
def save_job_record(job: Dict[str, Any], job_area: str, function_type: str,
|
||
page_no: int, records_file: str = "crawl_jobs_records.log") -> None:
|
||
"""保存单条职位记录到文件
|
||
|
||
Args:
|
||
job: 职位数据字典
|
||
job_area: 地区代码
|
||
function_type: 职能代码
|
||
page_no: 页码
|
||
records_file: 记录文件名
|
||
"""
|
||
try:
|
||
records_dir = os.path.dirname(os.path.abspath(__file__))
|
||
records_path = os.path.join(records_dir, records_file)
|
||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||
|
||
# 提取关键信息
|
||
job_id = get_job_id(job)
|
||
job_name = job.get('jobName') or job.get('name') or job.get('title') or ''
|
||
company_name = job.get('fullCompanyName') or job.get('companyName') or job.get('coName') or ''
|
||
salary = job.get('salary') or job.get('salaryDesc') or ''
|
||
|
||
# 构建记录信息
|
||
record = {
|
||
"timestamp": timestamp,
|
||
"page_no": page_no,
|
||
"job_area": job_area,
|
||
"function_type": function_type,
|
||
"job_id": job_id,
|
||
"job_name": job_name,
|
||
"company_name": company_name,
|
||
"salary": salary,
|
||
"job_data": job # 保存完整的职位数据
|
||
}
|
||
|
||
# 追加到文件（JSONL格式，每行一条JSON记录）
|
||
with open(records_path, "a", encoding="utf-8") as f:
|
||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||
|
||
except Exception as e:
|
||
logger.error(f"保存职位记录失败: {e}")
|
||
|
||
def save_crawl_stats(total_count: int, unique_count: int, total_count_api: Optional[int],
|
||
job_area: str, function_type: str, stats_file: str = "crawl_stats.log") -> None:
|
||
"""保存爬取统计信息到文件"""
|
||
try:
|
||
stats_dir = os.path.dirname(os.path.abspath(__file__))
|
||
stats_path = os.path.join(stats_dir, stats_file)
|
||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||
|
||
stats_info = {
|
||
"timestamp": timestamp,
|
||
"total_crawled": total_count,
|
||
"unique_count": unique_count,
|
||
"duplicate_count": total_count - unique_count,
|
||
"api_total_count": total_count_api,
|
||
"job_area": job_area,
|
||
"function_type": function_type
|
||
}
|
||
|
||
# 追加到文件
|
||
with open(stats_path, "a", encoding="utf-8") as f:
|
||
f.write(json.dumps(stats_info, ensure_ascii=False) + "\n")
|
||
|
||
logger.info(f"统计信息已保存到: {stats_path}")
|
||
except Exception as e:
|
||
logger.error(f"保存统计信息失败: {e}")
|
||
|
||
def fetch_service_params() -> Optional[Dict[str, str]]:
|
||
"""获取当天未使用的城市与职位并占用
|
||
|
||
Returns:
|
||
dict: {"job_area": str, "job_name": str} 或 None
|
||
"""
|
||
try:
|
||
url = f"{API_BASE_URL}/api/v1/keyword/available"
|
||
r = requests.get(url, params={"source": "qcwy", "limit": 1}, timeout=10)
|
||
if r.status_code != 200:
|
||
return None
|
||
js = r.json()
|
||
data = js.get("data") or {}
|
||
items = data.get("items") or []
|
||
if not items:
|
||
return None
|
||
item = items[0]
|
||
ids = [item.get("id")]
|
||
if ids and ids[0]:
|
||
try:
|
||
murl = f"{API_BASE_URL}/api/v1/keyword/mark-used"
|
||
requests.post(murl, json={"source": "qcwy", "ids": ids}, timeout=10)
|
||
except Exception:
|
||
pass
|
||
city = str(item.get("city", ""))
|
||
job = str(item.get("job", ""))
|
||
if not city or not job:
|
||
return None
|
||
return {"job_area": city, "job_name": job}
|
||
except Exception:
|
||
return None
|
||
|
||
@dataclass
|
||
class ProxyConfig:
|
||
"""代理配置"""
|
||
host: str
|
||
port: int
|
||
username: Optional[str] = None
|
||
password: Optional[str] = None
|
||
protocol: str = "http"
|
||
|
||
def to_url(self) -> str:
|
||
"""转换为代理URL"""
|
||
if self.username and self.password:
|
||
return f"{self.protocol}://{self.username}:{self.password}@{self.host}:{self.port}"
|
||
return f"{self.protocol}://{self.host}:{self.port}"
|
||
|
||
|
||
class ProxyManager:
|
||
"""代理IP管理器"""
|
||
|
||
def __init__(self, proxy_list: List[ProxyConfig] = None):
|
||
self.proxy_list = proxy_list or []
|
||
self.current_proxy_index = 0
|
||
self.failed_proxies = set()
|
||
|
||
def add_proxy(self, proxy: ProxyConfig):
|
||
"""添加代理"""
|
||
self.proxy_list.append(proxy)
|
||
|
||
def get_current_proxy(self) -> Optional[str]:
|
||
"""获取当前可用代理"""
|
||
return PROXY_URL
|
||
|
||
def mark_proxy_failed(self, proxy_url: str):
|
||
"""标记代理失败"""
|
||
for i, proxy in enumerate(self.proxy_list):
|
||
if proxy.to_url() == proxy_url:
|
||
self.failed_proxies.add(i)
|
||
logger.warning(f"代理 {proxy_url} 标记为失败")
|
||
break
|
||
|
||
def get_proxy_dict(self) -> Optional[Dict[str, str]]:
|
||
"""获取代理字典格式"""
|
||
return {"http": PROXY_URL, "https": PROXY_URL}
|
||
|
||
|
||
class RemoteReporter:
|
||
"""远程数据上报器"""
|
||
|
||
def __init__(self, report_url: str, proxy_url: Optional[str] = None):
|
||
self.report_url = report_url
|
||
self.public_host = API_PUBLIC_HOST
|
||
if proxy_url:
|
||
self.client = httpx.Client(timeout=30.0, proxy=proxy_url, verify=True)
|
||
else:
|
||
self.client = httpx.Client(timeout=30.0, verify=True)
|
||
def get_local_ip(self) -> str:
|
||
"""获取本地IP地址"""
|
||
try:
|
||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||
s.connect(("8.8.8.8", 80))
|
||
local_ip = s.getsockname()[0]
|
||
s.close()
|
||
return local_ip
|
||
except Exception:
|
||
return "127.0.0.1"
|
||
def report_data(self, data: List[Dict[str, Any]], data_type: str) -> bool:
|
||
"""上报数据到远程API"""
|
||
try:
|
||
# 计算实际数据条数
|
||
data_count = len(data) if isinstance(data, list) else 1
|
||
|
||
# 使用新的数据格式
|
||
universal_data = {
|
||
"data_list":data,
|
||
"data_type": data_type,
|
||
"platform": "qcwy"
|
||
}
|
||
|
||
headers = {
|
||
"accept": "application/json",
|
||
"Content-Type": "application/json",
|
||
'X-Forwarded-For': self.get_local_ip()
|
||
|
||
}
|
||
if self.public_host:
|
||
headers["Host"] = self.public_host
|
||
headers["X-Forwarded-Host"] = self.public_host
|
||
|
||
# 使用新的API端点
|
||
api_endpoint = f"{self.report_url}/api/v1/universal/data/batch-store-async"
|
||
|
||
response = requests.post(
|
||
api_endpoint,
|
||
json=universal_data,
|
||
headers=headers,
|
||
timeout=300
|
||
)
|
||
|
||
response_size = len(response.content) if hasattr(response, 'content') else 0
|
||
|
||
if 200 <= response.status_code < 300:
|
||
logger.info(f"数据上报成功: {data_type}, 上报条数: {data_count}, 响应大小: {response_size} 字节")
|
||
return True
|
||
else:
|
||
logger.error(f"数据上报失败: {response.status_code} - {response.text}, 上报条数: {data_count}, 响应大小: {response_size} 字节")
|
||
return False
|
||
|
||
except Exception as e:
|
||
logger.error(f"数据上报异常: {e}")
|
||
return False
|
||
|
||
def close(self):
|
||
"""关闭客户端"""
|
||
self.client.close()
|
||
|
||
|
||
class SignatureGenerator:
|
||
"""签名生成器"""
|
||
|
||
def __init__(self, sign_key: str):
|
||
self.sign_key = sign_key
|
||
|
||
def hmac_sha256(self, message: str) -> str:
|
||
"""使用HMAC-SHA256算法生成签名"""
|
||
key_bytes = self.sign_key.encode('utf-8')
|
||
message_bytes = message.encode('utf-8')
|
||
return hmac.new(key_bytes, message_bytes, hashlib.sha256).hexdigest()
|
||
|
||
def generate_signature(self, url_path: str, data: Dict[str, Any] = None) -> str:
|
||
"""生成请求签名"""
|
||
sign_message = url_path
|
||
|
||
if data:
|
||
# 将布尔值转换为字符串
|
||
data_copy = data.copy()
|
||
for key, value in data_copy.items():
|
||
if isinstance(value, bool):
|
||
data_copy[key] = "true" if value else "false"
|
||
|
||
# 添加请求体到签名消息
|
||
sign_message += json.dumps(data_copy, ensure_ascii=False, separators=(',', ':'))
|
||
|
||
return self.hmac_sha256(sign_message)
|
||
|
||
|
||
class JobCrawler:
|
||
"""51job职位爬虫主类"""
|
||
|
||
def __init__(self,
|
||
proxy_manager: ProxyManager = None,
|
||
remote_reporter: RemoteReporter = None,
|
||
sign_key: str = "abfc8f9dcf8c3f3d8aa294ac5f2cf2cc7767e5592590f39c3f503271dd68562b"):
|
||
|
||
self.proxy_manager = proxy_manager or ProxyManager()
|
||
self.remote_reporter = remote_reporter
|
||
self.signature_generator = SignatureGenerator(sign_key)
|
||
|
||
# API配置
|
||
self.base_url = "https://cupid.51job.com"
|
||
self.api_key = "51job"
|
||
|
||
# 基础请求头（不包含动态property）
|
||
self.base_headers = {
|
||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/6.8.0(0x16080000) NetType/WIFI MiniProgramEnv/Mac MacWechat/WMPF MacWechat/3.8.10(0x13080a10) XWEB/1227",
|
||
"Connection": "keep-alive",
|
||
"Accept": "*/*",
|
||
"Accept-Encoding": "gzip, deflate, br",
|
||
"Content-Type": "application/json",
|
||
"account-id": "",
|
||
"From-Domain": "51job_weixin_wxapp",
|
||
"xweb_xhr": "1",
|
||
"user-token": "",
|
||
"uuid": str(int(time.time() * 1000)) + str(random.randint(10000000, 99999999)),
|
||
"partner": "",
|
||
"timestamp": str(int(time.time() * 1000)),
|
||
"Sec-Fetch-Site": "cross-site",
|
||
"Sec-Fetch-Mode": "cors",
|
||
"Sec-Fetch-Dest": "empty",
|
||
"Referer": "https://servicewechat.com/wx1131e5c71e668b5d/391/page-frame.html",
|
||
"Accept-Language": "zh-CN,zh;q=0.9"
|
||
}
|
||
|
||
# Session复用 - httpx客户端
|
||
self.client = None
|
||
self._init_client()
|
||
|
||
# 加载城市和工作类型数据
|
||
self.cities_data = self._load_cities_data()
|
||
self.work_types_data = self._load_work_types_data()
|
||
|
||
def build_property(self, manual_login_method: str = "", page_code: str = "home|hotjob|jobfxlist") -> str:
|
||
"""动态生成property字段
|
||
|
||
Args:
|
||
manual_login_method: 手动登录方式
|
||
page_code: 页面代码
|
||
|
||
Returns:
|
||
URL编码的property JSON字符串
|
||
"""
|
||
import uuid
|
||
|
||
# 生成随机distinct_id（模拟微信小程序的设备ID）
|
||
distinct_id = str(int(time.time() * 1000)) + str(random.randint(100000, 999999))
|
||
|
||
property_data = {
|
||
"frompageUrl": "", # 来源页面URL
|
||
"pageUrl": "pages/index/index", # 当前页面URL
|
||
"isLogin": "否", # 是否登录
|
||
"accountid": "", # 账户ID
|
||
"resumeId": "", # 简历ID
|
||
"firstFrompageUrl": "", # 首次来源页面URL
|
||
"distinct_id": distinct_id, # 设备唯一标识
|
||
"pageCode": page_code, # 页面代码
|
||
"shortPageCode": page_code, # 短页面代码
|
||
"policyType": "推荐" # 策略类型
|
||
}
|
||
|
||
# 如果有手动登录方式，添加到property中
|
||
if manual_login_method:
|
||
property_data["manualLoginMethod"] = manual_login_method
|
||
|
||
# 转换为JSON字符串并URL编码
|
||
property_json = json.dumps(property_data, ensure_ascii=False, separators=(',', ':'))
|
||
property_encoded = quote(property_json)
|
||
|
||
|
||
|
||
return property_encoded
|
||
|
||
def _init_client(self):
|
||
"""初始化或重新初始化httpx客户端"""
|
||
if self.client:
|
||
self.client.close()
|
||
|
||
# 获取当前可用代理
|
||
proxies = self.proxy_manager.get_proxy_dict()
|
||
self.current_proxy = PROXY_URL
|
||
|
||
# 创建客户端配置（不包含headers，因为签名是动态的）
|
||
client_kwargs = {
|
||
"timeout": 30.0,
|
||
"verify": True,
|
||
"trust_env": False
|
||
}
|
||
|
||
# 启用代理配置（httpx使用proxy参数，传入字符串URL）
|
||
if self.current_proxy:
|
||
client_kwargs["proxy"] = self.current_proxy
|
||
logger.info(f"初始化客户端，使用代理: {self.current_proxy}")
|
||
|
||
self.client = httpx.Client(**client_kwargs)
|
||
|
||
def _switch_proxy_and_reinit(self):
|
||
"""切换代理并重新初始化客户端"""
|
||
if self.current_proxy:
|
||
logger.warning(f"标记代理失败并切换: {self.current_proxy}")
|
||
self.proxy_manager.mark_proxy_failed(self.current_proxy)
|
||
|
||
# 重新初始化客户端
|
||
self._init_client()
|
||
|
||
def _load_cities_data(self) -> Dict[str, Any]:
|
||
"""加载城市数据"""
|
||
try:
|
||
city_file = os.path.join(os.path.dirname(__file__), 'city.json')
|
||
with open(city_file, 'r', encoding='utf-8') as f:
|
||
return json.load(f)
|
||
except Exception as e:
|
||
logger.error(f"加载城市数据失败: {e}")
|
||
return {}
|
||
|
||
def _load_work_types_data(self) -> Dict[str, Any]:
|
||
"""加载工作类型数据"""
|
||
try:
|
||
work_file = os.path.join(os.path.dirname(__file__), 'work.json')
|
||
with open(work_file, 'r', encoding='utf-8') as f:
|
||
return json.load(f)
|
||
except Exception as e:
|
||
logger.error(f"加载工作类型数据失败: {e}")
|
||
return {}
|
||
|
||
def get_random_city(self) -> Dict[str, str]:
|
||
"""随机获取一个城市"""
|
||
try:
|
||
if not self.cities_data:
|
||
logger.warning("城市数据为空，使用默认城市")
|
||
return {"id": "020000", "value": "上海"}
|
||
all_cities: List[Dict[str, str]] = []
|
||
# 兼容多种结构：列表或包含 items 的字典
|
||
if isinstance(self.cities_data, list):
|
||
for item in self.cities_data:
|
||
code = item.get("city_code") or item.get("code")
|
||
name = item.get("city_name") or item.get("name")
|
||
if code and name:
|
||
all_cities.append({"id": str(code), "value": name})
|
||
elif isinstance(self.cities_data, dict):
|
||
for item in self.cities_data.get("items", []):
|
||
code = item.get("city_code") or item.get("code")
|
||
name = item.get("city_name") or item.get("name")
|
||
if code and name:
|
||
all_cities.append({"id": str(code), "value": name})
|
||
if not all_cities:
|
||
logger.warning("未找到城市数据，使用默认城市")
|
||
return {"id": "020000", "value": "上海"}
|
||
return random.choice(all_cities)
|
||
except Exception:
|
||
return {"id": "020000", "value": "上海"}
|
||
|
||
def get_city_name_by_code(self, city_code: str) -> str:
|
||
"""根据城市代码获取城市名称
|
||
|
||
Args:
|
||
city_code: 城市代码
|
||
|
||
Returns:
|
||
str: 城市名称，未找到时返回代码
|
||
"""
|
||
try:
|
||
if not city_code:
|
||
return city_code
|
||
if not self.cities_data:
|
||
return city_code
|
||
# 兼容列表和字典格式
|
||
items = self.cities_data if isinstance(self.cities_data, list) else self.cities_data.get("items", [])
|
||
for item in items:
|
||
if str(item.get("city_code")) == str(city_code):
|
||
return item.get("city_name") or city_code
|
||
return city_code
|
||
except Exception:
|
||
return city_code
|
||
|
||
def get_city_pinyin_by_code(self, city_code: str) -> str:
|
||
"""根据城市代码获取拼音
|
||
|
||
Args:
|
||
city_code: 城市代码
|
||
|
||
Returns:
|
||
str: 城市对应的拼音，未找到时返回默认值
|
||
"""
|
||
try:
|
||
if not city_code:
|
||
return "hanzhong"
|
||
if not self.cities_data:
|
||
return "hanzhong"
|
||
# 兼容列表和字典格式
|
||
items = self.cities_data if isinstance(self.cities_data, list) else self.cities_data.get("items", [])
|
||
for item in items:
|
||
if str(item.get("city_code")) == str(city_code):
|
||
return item.get("pinyin") or "hanzhong"
|
||
return "hanzhong"
|
||
except Exception:
|
||
return "hanzhong"
|
||
|
||
def get_function_name_by_code(self, function_code: str) -> str:
|
||
"""根据职能代码获取职能名称
|
||
|
||
Args:
|
||
function_code: 职能代码
|
||
|
||
Returns:
|
||
str: 职能名称，未找到时返回代码
|
||
"""
|
||
try:
|
||
if not function_code:
|
||
return function_code
|
||
if isinstance(self.work_types_data, dict):
|
||
for cat in self.work_types_data.get('items', []):
|
||
for sub in cat.get('items', []):
|
||
for it in sub.get('items', []):
|
||
code = it.get('code') or it.get('funcTypeCode')
|
||
name = it.get('value') or it.get('name')
|
||
if str(code) == str(function_code):
|
||
return name or function_code
|
||
return function_code
|
||
except Exception:
|
||
return function_code
|
||
|
||
def get_random_work_type(self) -> Dict[str, str]:
|
||
"""随机获取一个工作类型"""
|
||
try:
|
||
items = []
|
||
if isinstance(self.work_types_data, dict):
|
||
for cat in self.work_types_data.get('items', []):
|
||
for sub in cat.get('items', []):
|
||
for it in sub.get('items', []):
|
||
code = it.get('code') or it.get('funcTypeCode')
|
||
name = it.get('value') or it.get('name')
|
||
if code and name:
|
||
items.append({'code': str(code), 'value': name})
|
||
if not items:
|
||
logger.warning("工作类型数据为空，使用默认工作类型")
|
||
return {"code": "A0N7", "value": "数据分析"}
|
||
return random.choice(items)
|
||
except Exception:
|
||
return {"code": "A0N7", "value": "数据分析"}
|
||
|
||
def find_function_code_by_name(self, name: str) -> Optional[str]:
|
||
"""根据工作类型名称查找职能代码
|
||
|
||
Args:
|
||
name: 工作类型中文名称
|
||
|
||
Returns:
|
||
职能代码或None
|
||
"""
|
||
try:
|
||
if not name:
|
||
return None
|
||
if isinstance(self.work_types_data, dict):
|
||
for cat in self.work_types_data.get('items', []):
|
||
for sub in cat.get('items', []):
|
||
for it in sub.get('items', []):
|
||
val = it.get('value') or it.get('name')
|
||
code = it.get('code') or it.get('funcTypeCode')
|
||
if isinstance(val, str) and isinstance(code, str) and val.strip() == name.strip():
|
||
return code
|
||
except Exception:
|
||
return None
|
||
return None
|
||
|
||
def _normalize_job_area(self, code: Optional[str]) -> str:
|
||
code_str = str(code or "").strip()
|
||
return code_str if len(code_str) == 6 and code_str.isdigit() else "020000"
|
||
|
||
def _normalize_function_type(self, code: Optional[str]) -> str:
|
||
code_str = str(code or "").strip().upper()
|
||
return code_str if code_str.startswith("A0") else "A0N7"
|
||
|
||
def _make_request(self, url: str, data: Dict[str, Any] = None, headers: Dict[str, str] = None, max_retries: int = 3, method: str = "POST") -> Optional[Dict[str, Any]]:
|
||
"""发送HTTP请求，支持session复用和智能代理切换"""
|
||
for attempt in range(max_retries):
|
||
try:
|
||
# 检查客户端是否需要重新初始化
|
||
if not self.client or self.client.is_closed:
|
||
self._init_client()
|
||
|
||
# 使用传入的headers或默认headers
|
||
request_headers = headers or self.base_headers
|
||
|
||
# 根据方法类型发送请求
|
||
if method.upper() == "GET":
|
||
response = self.client.get(url, headers=request_headers)
|
||
else:
|
||
response = self.client.post(url, headers=request_headers, json=data)
|
||
|
||
if response.status_code == 200:
|
||
result = response.json()
|
||
logger.info("请求参数 method={} url={} status={} resp_size={}", method.upper(), url, response.status_code, len(response.content))
|
||
return result
|
||
else:
|
||
logger.warning(f"请求失败: {response.status_code} - {response.text}")
|
||
|
||
# 如果是代理相关的状态码，切换代理
|
||
if response.status_code in [407, 502, 503, 504, 403]:
|
||
logger.warning(f"检测到代理问题，状态码: {response.status_code}")
|
||
self._switch_proxy_and_reinit()
|
||
continue
|
||
|
||
# 其他错误，等待后重试
|
||
if attempt < max_retries - 1:
|
||
time.sleep(2 ** attempt)
|
||
|
||
except Exception as e:
|
||
logger.error(f"请求异常 (尝试 {attempt + 1}/{max_retries}): {e}")
|
||
|
||
# 检查是否是代理相关异常
|
||
error_str = str(e).lower()
|
||
if any(keyword in error_str for keyword in ['proxy', 'connection', 'timeout', 'ssl', 'certificate']):
|
||
logger.warning(f"检测到代理相关异常: {e}")
|
||
self._switch_proxy_and_reinit()
|
||
continue
|
||
|
||
# 非代理异常，等待后重试
|
||
if attempt < max_retries - 1:
|
||
time.sleep(2 ** attempt) # 指数退避
|
||
|
||
logger.error(f"所有重试失败，请求终止")
|
||
return None
|
||
|
||
def get_recommend_jobs(self,
|
||
page_no: int = 1,
|
||
page_size: int = 10,
|
||
job_area: str = "020000",
|
||
function_type: str = "A0N7",
|
||
seen_job_ids: Optional[set] = None) -> Tuple[List[Dict[str, Any]], int]:
|
||
"""获取推荐职位
|
||
|
||
Returns:
|
||
tuple: (职位列表, totalCount总数)
|
||
"""
|
||
logger.info(f"获取推荐职位: 页码={page_no}, 页大小={page_size}, 地区={job_area}, 工作类型={function_type}")
|
||
|
||
timestamp = int(time.time())
|
||
|
||
# 请求参数
|
||
data = {
|
||
"pageNo": page_no,
|
||
"pageSize": page_size,
|
||
"specialPageCode": True,
|
||
"isTouristMode": True,
|
||
"type": "recommend",
|
||
"jobArea": self._normalize_job_area(job_area),
|
||
"personAsLabel": "",
|
||
"functionType": self._normalize_function_type(function_type)
|
||
}
|
||
|
||
# 构建URL路径
|
||
api_path = "open/noauth/recommend/job-tab-dynamic-wx-mini"
|
||
url_path = f"/{api_path}?api_key={self.api_key}×tamp={timestamp}"
|
||
full_url = f"{self.base_url}{url_path}"
|
||
|
||
# 生成签名
|
||
signature = self.signature_generator.generate_signature(url_path, data)
|
||
|
||
# 动态生成property字段
|
||
property_value = self.build_property(page_code="home|hotjob|jobfxlist")
|
||
|
||
# 添加签名和动态property到请求头
|
||
headers = self.base_headers.copy()
|
||
headers["sign"] = signature
|
||
headers["property"] = property_value
|
||
|
||
# 将布尔值转换为字符串
|
||
for key, value in data.items():
|
||
if isinstance(value, bool):
|
||
data[key] = "true" if value else "false"
|
||
# 先尝试 GET，再回退 POST
|
||
signature_get = self.signature_generator.generate_signature(url_path)
|
||
headers_get = self.base_headers.copy()
|
||
headers_get["sign"] = signature_get
|
||
headers_get["property"] = property_value
|
||
headers_get["Content-Type"] = "application/x-www-form-urlencoded"
|
||
response_data = self._make_request(full_url, None, headers_get, method="GET")
|
||
if not (response_data and response_data.get("status") in ['1', 1]):
|
||
response_data = self._make_request(full_url, data, headers, method="POST")
|
||
if not response_data:
|
||
return [], 0
|
||
|
||
# 解析响应
|
||
try:
|
||
if response_data.get("status") in [1, "1"] and response_data.get("resultbody"):
|
||
job_list = response_data["resultbody"].get("jobList", {})
|
||
jobs = job_list.get("items", [])
|
||
# 提取totalCount
|
||
total_count_raw = job_list.get("totalCount") or job_list.get("totalcount")
|
||
total_count = 0
|
||
if total_count_raw is not None:
|
||
if isinstance(total_count_raw, str):
|
||
try:
|
||
total_count = int(total_count_raw)
|
||
except (ValueError, TypeError):
|
||
total_count = 0
|
||
elif isinstance(total_count_raw, (int, float)):
|
||
total_count = int(total_count_raw)
|
||
else:
|
||
total_count = 0
|
||
|
||
all_jobs = jobs
|
||
# 获取城市名称和职能名称用于日志
|
||
city_name = self.get_city_name_by_code(str(job_area))
|
||
function_name = self.get_function_name_by_code(str(function_type))
|
||
|
||
# 🔍 调试：打印每页返回的前5个 jobId，验证分页是否正确
|
||
job_ids_preview = [get_job_id(job) for job in jobs[:5]]
|
||
logger.info(f"🔍 第 {page_no} 页 jobIds 预览: {job_ids_preview}")
|
||
|
||
# 内存去重：过滤掉已爬取的职位
|
||
if seen_job_ids is not None:
|
||
unique_jobs = []
|
||
duplicate_count = 0
|
||
for job in all_jobs:
|
||
job_id = get_job_id(job)
|
||
if job_id and job_id in seen_job_ids:
|
||
duplicate_count += 1
|
||
continue
|
||
if job_id:
|
||
seen_job_ids.add(job_id)
|
||
unique_jobs.append(job)
|
||
all_jobs = unique_jobs
|
||
if duplicate_count > 0:
|
||
logger.warning(f"第 {page_no} 页发现 {duplicate_count} 个重复职位（已跳过）")
|
||
|
||
logger.info(f"第 {page_no} 页成功获取 {len(jobs)} 个职位（去重后: {len(all_jobs)}） | API返回totalCount: {total_count} (原始值: {total_count_raw}) | 地区: {city_name}({job_area}) | 职能: {function_name}({function_type})")
|
||
|
||
# 保存每条职位记录到文件
|
||
if all_jobs:
|
||
for job in all_jobs:
|
||
save_job_record(job, job_area, function_type, page_no)
|
||
|
||
# 上报数据（只上报去重后的数据）
|
||
if all_jobs:
|
||
jobs_with_details = []
|
||
for i, job in enumerate(all_jobs, 1):
|
||
try:
|
||
job_id = job.get('jobId') or job.get('id') or job.get('job_id')
|
||
company_id = job.get('coId') or job.get('companyId')
|
||
target_job = job.copy()
|
||
city_pinyin = self.get_city_pinyin_by_code(str(job_area))
|
||
target_job["jobHref"] = f"https://jobs.51job.com/{city_pinyin}/{job_id}.html"
|
||
|
||
# 获取职位详情
|
||
if job_id:
|
||
try:
|
||
job_detail = self.get_job_detail(str(job_id))
|
||
if job_detail:
|
||
target_job['jobDescribe'] = job_detail.get('detailJobInfo', {}).get("jobDescribe")
|
||
if not company_id:
|
||
company_id = job_detail.get('coId') or job_detail.get('companyId')
|
||
except Exception as e:
|
||
logger.warning(f"获取职位详情失败 job_id={job_id}: {e}")
|
||
|
||
# 获取公司信息
|
||
if company_id:
|
||
try:
|
||
company_info = self.get_company_info(str(company_id))
|
||
if company_info:
|
||
target_job['company_info'] = company_info
|
||
target_job['company_desc'] = company_info.get("coinfo",{}).get("coinfo")
|
||
target_job['companyHref'] = company_info.get("share",{}).get("weixinshareurl")
|
||
except Exception as e:
|
||
logger.warning(f"获取公司信息失败 company_id={company_id}: {e}")
|
||
|
||
jobs_with_details.append(target_job)
|
||
|
||
# 在获取详情之间添加延迟（最后一个元素不需要延迟）
|
||
if i < len(all_jobs):
|
||
sleep_random_between()
|
||
except Exception as e:
|
||
logger.error(f"处理职位数据失败: {e}")
|
||
# 即使处理失败，也继续处理下一条
|
||
continue
|
||
|
||
if self.remote_reporter and jobs_with_details:
|
||
logger.info(f"准备上报第 {page_no} 页数据，包含详细信息的数据条数: {len(jobs_with_details)}")
|
||
self.remote_reporter.report_data(jobs_with_details, "job")
|
||
|
||
return jobs, total_count
|
||
else:
|
||
logger.warning(f"API返回错误: {response_data}")
|
||
return [], 0
|
||
|
||
except Exception as e:
|
||
import traceback
|
||
traceback.print_exc()
|
||
logger.error(f"解析响应失败: {e}")
|
||
return [], 0
|
||
|
||
def get_company_jobs(self,
|
||
page_no: int = 1,
|
||
page_size: int = 30,
|
||
co_id: str = "",
|
||
job_area: str = "",
|
||
function: str = "",
|
||
salary_type: str = "") -> List[Dict[str, Any]]:
|
||
"""获取公司职位"""
|
||
url = "https://cupid.51job.com/open/noauth/jobs/company"
|
||
|
||
params = {
|
||
"api_key": "51job",
|
||
"timestamp": int(time.time())
|
||
}
|
||
|
||
data = {
|
||
"pageNum": page_no,
|
||
"pageSize": page_size,
|
||
"coId": co_id,
|
||
"jobArea": job_area,
|
||
"function": function,
|
||
"salaryType": salary_type,
|
||
"scene": 14,
|
||
"requestId": ""
|
||
}
|
||
|
||
# 构建完整URL
|
||
full_url = f"{url}?api_key={params['api_key']}×tamp={params['timestamp']}"
|
||
|
||
# 生成签名（包含查询参数）
|
||
url_with_params = f"/open/noauth/jobs/company?api_key={params['api_key']}×tamp={params['timestamp']}"
|
||
signature = self.signature_generator.generate_signature(url_with_params, data)
|
||
|
||
# 构建property字段
|
||
property_value = self.build_property(page_code="companyDetail|job|joblist")
|
||
|
||
# 添加签名和动态property到请求头
|
||
headers = self.base_headers.copy()
|
||
headers["sign"] = signature
|
||
headers["property"] = property_value
|
||
|
||
try:
|
||
response_data = self._make_request(full_url, data, headers)
|
||
if not response_data:
|
||
signature_get = self.signature_generator.generate_signature(url_with_params)
|
||
headers_get = self.base_headers.copy()
|
||
headers_get["sign"] = signature_get
|
||
headers_get["property"] = property_value
|
||
response_data = self._make_request(full_url, None, headers_get, method="GET")
|
||
|
||
if response_data and response_data.get('status') in ['1', 1]:
|
||
result_list = response_data.get('resultbody', {}).get('items', [])
|
||
all_jobs = result_list
|
||
if result_list:
|
||
logger.info(f"成功获取 {len(result_list)} 个公司职位")
|
||
|
||
else:
|
||
logger.warning("API返回成功但职位列表为空")
|
||
|
||
return result_list
|
||
else:
|
||
logger.error(f"获取公司职位失败: {response_data}")
|
||
return []
|
||
|
||
except Exception as e:
|
||
logger.error(f"获取公司职位时发生异常: {str(e)}")
|
||
|
||
return []
|
||
|
||
def get_job_detail(self, job_id: str) -> Dict[str, Any]:
|
||
"""获取职位详情"""
|
||
|
||
|
||
timestamp = int(time.time())
|
||
|
||
# 构建URL
|
||
api_path = f"open/noauth/jobs/detail/base/{job_id}"
|
||
url_path = f"/{api_path}?api_key={self.api_key}×tamp={timestamp}"
|
||
full_url = f"{self.base_url}{url_path}"
|
||
|
||
# 生成签名
|
||
signature = self.signature_generator.generate_signature(url_path)
|
||
|
||
# 构建property字段
|
||
property_value = self.build_property(page_code="pages/jobs/jobdetail/jobdetail")
|
||
|
||
# 添加签名和动态property到请求头
|
||
headers = self.base_headers.copy()
|
||
headers["sign"] = signature
|
||
headers["property"] = property_value
|
||
headers["Content-Type"] = "application/x-www-form-urlencoded"
|
||
# 可选登录头，开放接口无需
|
||
|
||
try:
|
||
# 使用GET请求获取职位详情
|
||
response_data = self._make_request(full_url, None, headers, method="GET")
|
||
|
||
if response_data and response_data.get('status') in ['1', 1]:
|
||
job_detail = response_data.get('resultbody', {})
|
||
|
||
if job_detail:
|
||
|
||
return job_detail
|
||
else:
|
||
logger.warning(f"职位详情为空: {job_id}")
|
||
return {}
|
||
else:
|
||
logger.error(f"获取职位详情失败: {job_id}, 响应: {response_data}")
|
||
return {}
|
||
|
||
except Exception as e:
|
||
logger.error(f"获取职位详情时发生异常: {job_id}, 错误: {str(e)}")
|
||
return {}
|
||
|
||
def get_company_info(self, company_id: str) -> Dict[str, Any]:
|
||
"""获取公司详情信息"""
|
||
|
||
timestamp = int(time.time())
|
||
|
||
# 构建URL和参数
|
||
api_path = "open/noauth/company-info/info-data"
|
||
params = {
|
||
"companyId": company_id,
|
||
"colorOne": "#ffffff",
|
||
"colorTwo": "#ffffffcc"
|
||
}
|
||
|
||
# 构建完整URL路径（包含查询参数）
|
||
url_path = f"/{api_path}?api_key={self.api_key}×tamp={timestamp}"
|
||
|
||
# 添加业务参数到URL
|
||
param_str = "&".join([f"{k}={quote(str(v))}" for k, v in params.items()])
|
||
url_path += f"&{param_str}"
|
||
full_url = f"{self.base_url}{url_path}"
|
||
|
||
# 生成签名（GET请求，签名只包含URL路径）
|
||
signature = self.signature_generator.generate_signature(url_path)
|
||
|
||
# 构建property字段
|
||
property_value = self.build_property(page_code="companyDetail|company|companyinfo")
|
||
|
||
# 添加签名和动态property到请求头
|
||
headers = self.base_headers.copy()
|
||
headers["sign"] = signature
|
||
headers["property"] = property_value
|
||
headers["Content-Type"] = "application/x-www-form-urlencoded"
|
||
|
||
try:
|
||
# 使用GET请求获取公司详情
|
||
response_data = self._make_request(full_url, None, headers, method="GET")
|
||
|
||
if response_data and response_data.get('status') in ['1', 1]:
|
||
company_info = response_data.get('resultbody', {})
|
||
|
||
if company_info:
|
||
|
||
return company_info
|
||
else:
|
||
logger.warning(f"公司详情为空: {company_id}")
|
||
return {}
|
||
else:
|
||
logger.error(f"获取公司详情失败: {response_data}")
|
||
return {}
|
||
|
||
except Exception as e:
|
||
logger.error(f"获取公司详情时发生异常: {str(e)}")
|
||
return {}
|
||
|
||
def crawl_multiple_pages(self,
|
||
max_pages: int = 5,
|
||
page_size: int = 10,
|
||
job_area: str = "020000",
|
||
function_type: str = "A0N7",
|
||
delay: float = 1.0) -> List[Dict[str, Any]]:
|
||
"""爬取指定页数的数据，根据totalCount动态计算页数"""
|
||
all_jobs = []
|
||
total_count = None
|
||
calculated_max_pages = max_pages
|
||
page = 1
|
||
actual_pages_crawled = 0 # 记录实际爬取的页数
|
||
|
||
# 内存去重：维护已爬取的job_id集合，避免重复获取详情和上报
|
||
seen_job_ids = set()
|
||
|
||
# 获取城市名称和职能名称用于日志
|
||
city_name = self.get_city_name_by_code(str(job_area))
|
||
function_name = self.get_function_name_by_code(str(function_type))
|
||
|
||
logger.info("=" * 80)
|
||
logger.info(f"开始爬取推荐职位 | 最大页数: {max_pages} | 每页大小: {page_size} | 地区: {city_name}({job_area}) | 职能: {function_name}({function_type})")
|
||
logger.info("=" * 80)
|
||
|
||
while page <= calculated_max_pages:
|
||
logger.info(f"\n>>> 正在爬取第 {page}/{calculated_max_pages} 页...")
|
||
try:
|
||
self._init_client()
|
||
except Exception:
|
||
pass
|
||
|
||
jobs, current_total_count = self.get_recommend_jobs(
|
||
page_no=page,
|
||
page_size=page_size,
|
||
job_area=job_area,
|
||
function_type=function_type,
|
||
seen_job_ids=seen_job_ids # 传入已爬取的job_id集合，用于去重
|
||
)
|
||
|
||
# 记录实际爬取的页数
|
||
actual_pages_crawled += 1
|
||
|
||
# 首次获取时，根据totalCount动态计算需要爬取的页数
|
||
if total_count is None and current_total_count > 0:
|
||
total_count = current_total_count
|
||
# 根据totalCount计算实际需要的页数（向上取整）
|
||
calculated_max_pages = min(max_pages, (total_count + page_size - 1) // page_size)
|
||
logger.info(f"检测到API返回totalCount: {total_count} | 计算需要爬取: {calculated_max_pages} 页 (每页 {page_size} 条) | 预计最大数据: {calculated_max_pages * page_size} 条")
|
||
|
||
if jobs:
|
||
all_jobs.extend(jobs)
|
||
logger.info(f"第 {page} 页获取到 {len(jobs)} 个职位 | 累计: {len(all_jobs)} 个")
|
||
else:
|
||
logger.warning(f"第 {page} 页没有获取到职位数据，提前结束")
|
||
# 如果某页没有数据，提前结束
|
||
break
|
||
|
||
# 如果已获取的数据量达到或超过totalCount，提前结束
|
||
if total_count and len(all_jobs) >= total_count:
|
||
logger.info(f"已获取所有数据 ({len(all_jobs)}/{total_count})，提前结束爬取")
|
||
break
|
||
|
||
# 延迟避免被封（在每页请求之间添加休眠）
|
||
if page < calculated_max_pages:
|
||
sleep_time = sleep_random_between()
|
||
logger.info(f"休眠 {sleep_time:.2f} 秒后继续...")
|
||
|
||
page += 1
|
||
|
||
# 统计和去重
|
||
total_crawled = len(all_jobs)
|
||
unique_jobs = deduplicate_jobs(all_jobs)
|
||
unique_count = len(unique_jobs)
|
||
duplicate_count = total_crawled - unique_count
|
||
|
||
# 打印详细统计信息
|
||
logger.info("\n" + "=" * 80)
|
||
logger.info("爬取统计信息:")
|
||
logger.info(f" 总抓取数量: {total_crawled} 条")
|
||
logger.info(f" 去重后数量: {unique_count} 条")
|
||
logger.info(f" 重复数量: {duplicate_count} 条")
|
||
logger.info(f" API返回totalCount: {total_count if total_count is not None else '未知'} 条")
|
||
logger.info(f" 实际爬取页数: {actual_pages_crawled} 页")
|
||
logger.info(f" 地区: {city_name}({job_area})")
|
||
logger.info(f" 职能: {function_name}({function_type})")
|
||
if total_count and total_count > 0:
|
||
coverage_rate = (unique_count / total_count * 100)
|
||
logger.info(f" 覆盖率: {coverage_rate:.2f}% ({unique_count}/{total_count})")
|
||
if unique_count < total_count:
|
||
logger.warning(f" 注意: 实际抓取数量({unique_count}) < API返回总数({total_count})，可能存在分页问题或数据过滤")
|
||
elif total_count == 0:
|
||
logger.warning(f"注意: API返回totalCount为0，可能是该地区/职能组合下没有数据")
|
||
logger.info("=" * 80 + "\n")
|
||
|
||
# 保存统计信息到文件
|
||
save_crawl_stats(
|
||
total_count=total_crawled,
|
||
unique_count=unique_count,
|
||
total_count_api=total_count,
|
||
job_area=job_area,
|
||
function_type=function_type
|
||
)
|
||
|
||
# 记录保存的文件路径
|
||
records_dir = os.path.dirname(os.path.abspath(__file__))
|
||
records_path = os.path.join(records_dir, "crawl_jobs_records.log")
|
||
logger.info(f"职位记录已保存到: {records_path} (共 {unique_count} 条记录)")
|
||
|
||
return unique_jobs
|
||
|
||
def crawl_company_jobs_multiple_pages(self,
|
||
co_id: str,
|
||
max_pages: int = 5,
|
||
page_size: int = 30,
|
||
job_area: str = "",
|
||
function: str = "",
|
||
salary_type: str = "",
|
||
delay: float = 1.0) -> List[Dict[str, Any]]:
|
||
"""爬取指定页数的公司职位数据"""
|
||
all_jobs = []
|
||
|
||
for page in range(1, max_pages + 1):
|
||
logger.info(f"正在爬取公司 {co_id} 第 {page} 页职位...")
|
||
try:
|
||
self._init_client()
|
||
except Exception:
|
||
pass
|
||
|
||
jobs = self.get_company_jobs(
|
||
page_no=page,
|
||
page_size=page_size,
|
||
co_id=co_id,
|
||
job_area=job_area,
|
||
function=function,
|
||
salary_type=salary_type
|
||
)
|
||
|
||
if jobs:
|
||
all_jobs.extend(jobs)
|
||
logger.info(f"第 {page} 页获取到 {len(jobs)} 个职位")
|
||
else:
|
||
logger.warning(f"第 {page} 页没有获取到职位数据")
|
||
|
||
# 延迟避免被封
|
||
if page < max_pages:
|
||
sleep_random_between()
|
||
|
||
logger.info(f"总共获取到 {len(all_jobs)} 个公司职位")
|
||
|
||
return all_jobs
|
||
|
||
|
||
def close(self):
|
||
"""关闭资源"""
|
||
# 关闭httpx客户端
|
||
if self.client and not self.client.is_closed:
|
||
self.client.close()
|
||
logger.info("httpx客户端已关闭")
|
||
|
||
# 关闭远程报告器
|
||
if self.remote_reporter:
|
||
self.remote_reporter.close()
|
||
|
||
|
||
def crawl_recommend_jobs_main(max_pages: int = 10, page_size: int = 15, job_area: str = None, function_type: str = None, delay: float = 2.0):
|
||
"""推荐职位爬取主函数
|
||
|
||
Args:
|
||
max_pages: 最大爬取页数
|
||
page_size: 每页数量
|
||
job_area: 地区代码，为None时随机选择
|
||
function_type: 工作类型代码，为None时随机选择
|
||
delay: 页面间隔时间
|
||
|
||
Returns:
|
||
List[Dict]: 职位列表
|
||
"""
|
||
# 1. 配置代理（可选）
|
||
proxy_manager = ProxyManager()
|
||
username = os.getenv('PROXY_USERNAME', 't13319619426654')
|
||
password = os.getenv('PROXY_PASSWORD', 'ln8aj9nl')
|
||
tunnel = os.getenv('PROXY_TUNNEL', 's432.kdltps.com:15818')
|
||
|
||
# 解析隧道地址和端口
|
||
if tunnel and ':' in tunnel:
|
||
host, port_str = tunnel.split(':')
|
||
port = int(port_str)
|
||
# 使用环境变量中的代理信息创建代理配置
|
||
proxy_config = ProxyConfig(host, port, username, password)
|
||
proxy_manager.add_proxy(proxy_config)
|
||
logger.info(f"已配置代理: {proxy_config.to_url()}")
|
||
else:
|
||
logger.warning("未找到有效的隧道配置，将使用本地代理")
|
||
proxy_manager.add_proxy(ProxyConfig("127.0.0.1", 8080)) # 添加本地代理作为备选
|
||
|
||
# 2. 配置远程上报（可选）
|
||
|
||
# 3. 创建爬虫实例
|
||
remote_reporter = RemoteReporter(
|
||
report_url=API_BASE_URL,
|
||
# proxy_url=proxy_manager.get_current_proxy()
|
||
)
|
||
crawler = JobCrawler(
|
||
proxy_manager=proxy_manager,
|
||
remote_reporter=remote_reporter
|
||
)
|
||
|
||
try:
|
||
svc = fetch_service_params()
|
||
if svc:
|
||
job_area = svc["job_area"]
|
||
code = crawler.find_function_code_by_name(svc["job_name"]) if function_type is None else function_type
|
||
if code:
|
||
function_type = code
|
||
# 回退随机选择
|
||
if job_area is None:
|
||
random_city = crawler.get_random_city()
|
||
job_area = random_city['id']
|
||
logger.info(f"随机选择城市: {random_city['value']} ({random_city['id']})")
|
||
if function_type is None:
|
||
random_work_type = crawler.get_random_work_type()
|
||
function_type = random_work_type['code']
|
||
logger.info(f"随机选择工作类型: {random_work_type['value']} ({random_work_type['code']})")
|
||
|
||
logger.info(f"开始爬取推荐职位，最多 {max_pages} 页")
|
||
|
||
all_jobs = crawler.crawl_multiple_pages(
|
||
max_pages=max_pages,
|
||
page_size=page_size,
|
||
job_area=job_area,
|
||
function_type=function_type,
|
||
delay=delay
|
||
)
|
||
|
||
# 显示结果
|
||
if all_jobs:
|
||
logger.info(f"推荐职位爬取完成！共找到 {len(all_jobs)} 个职位")
|
||
else:
|
||
logger.warning("未找到任何推荐职位信息")
|
||
|
||
return all_jobs
|
||
|
||
except KeyboardInterrupt:
|
||
logger.info("用户中断推荐职位爬取")
|
||
return []
|
||
except Exception as e:
|
||
logger.error(f"推荐职位爬取过程中发生错误: {e}")
|
||
return []
|
||
finally:
|
||
crawler.close()
|
||
|
||
|
||
def crawl_company_jobs_main(co_id: str, max_pages: int = 5, page_size: int = 30, job_area: str = "", function: str = "", salary_type: str = "", delay: float = 2.0):
|
||
"""公司职位爬取主函数
|
||
|
||
Args:
|
||
co_id: 公司ID
|
||
max_pages: 最大爬取页数
|
||
page_size: 每页数量
|
||
job_area: 地区代码
|
||
function: 职能代码
|
||
salary_type: 薪资类型
|
||
delay: 页面间隔时间
|
||
|
||
Returns:
|
||
List[Dict]: 职位列表
|
||
"""
|
||
# 1. 配置代理（可选）
|
||
proxy_manager = ProxyManager()
|
||
# proxy_manager.add_proxy(ProxyConfig("127.0.0.1", 8080)) # 添加代理
|
||
|
||
# 2. 配置远程上报（可选）
|
||
remote_reporter = RemoteReporter(
|
||
report_url=API_BASE_URL,
|
||
proxy_url=proxy_manager.get_current_proxy()
|
||
)
|
||
|
||
# 3. 创建爬虫实例
|
||
crawler = JobCrawler(
|
||
proxy_manager=proxy_manager,
|
||
remote_reporter = remote_reporter
|
||
)
|
||
|
||
try:
|
||
logger.info(f"开始爬取公司 {co_id} 的职位，最多 {max_pages} 页")
|
||
|
||
all_jobs = crawler.crawl_company_jobs_multiple_pages(
|
||
co_id=co_id,
|
||
max_pages=max_pages,
|
||
page_size=page_size,
|
||
job_area=job_area,
|
||
function=function,
|
||
salary_type=salary_type,
|
||
delay=delay
|
||
)
|
||
return all_jobs
|
||
|
||
except KeyboardInterrupt:
|
||
logger.info("用户中断公司职位爬取")
|
||
return []
|
||
except Exception as e:
|
||
logger.error(f"公司职位爬取过程中发生错误: {e}")
|
||
return []
|
||
finally:
|
||
crawler.close()
|
||
|
||
|
||
def main():
|
||
"""主函数示例 - 演示两个爬取函数的用法"""
|
||
# 使用更大的默认值，根据totalCount会动态调整实际爬取的页数
|
||
recommend_jobs = crawl_recommend_jobs_main(max_pages=50, page_size=20)
|
||
|
||
if __name__ == "__main__":
|
||
while True:
|
||
main()
|
||
sleep_random_between()
|