54 lines
1.5 KiB
Python
54 lines
1.5 KiB
Python
"""
|
|
智联招聘 独立公司爬虫入口
|
|
|
|
从 pending_company 队列获取待爬取的智联公司，
|
|
逐个调用 GetCompanyDetail 获取详情并上传。
|
|
|
|
启动:
|
|
python -m spiderJobs.platforms.zhilian.company_main
|
|
|
|
环境变量:
|
|
API_BASE_URL 后端地址 (默认 http://124.222.106.226:9999)
|
|
COMPANY_BATCH_SIZE 每批获取公司数 (默认 10)
|
|
SLEEP_MIN_SECONDS 最小延迟秒数 (默认 10)
|
|
SLEEP_MAX_SECONDS 最大延迟秒数 (默认 20)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
|
|
_project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
|
|
if _project_root not in sys.path:
|
|
sys.path.insert(0, _project_root)
|
|
|
|
from spiderJobs.core.base import BaseFetcher
|
|
from spiderJobs.platforms.zhilian.api import GetCompanyDetail
|
|
from spiderJobs.platforms.zhilian.client import ZhilianClient, create_cgate_client
|
|
from spiderJobs.runner.company_loop import run_company_loop
|
|
|
|
|
|
def create_company_fetcher(company_id: str, http_client: ZhilianClient) -> BaseFetcher:
|
|
"""创建智联公司详情 fetcher"""
|
|
return GetCompanyDetail(number=company_id, client=http_client)
|
|
|
|
|
|
def main():
|
|
client_kwargs = {}
|
|
|
|
proxy = os.environ.get("PROXY_URL", "")
|
|
if proxy:
|
|
client_kwargs["proxy"] = proxy
|
|
|
|
run_company_loop(
|
|
platform="zhilian",
|
|
create_company_fetcher=create_company_fetcher,
|
|
create_client_fn=create_cgate_client,
|
|
client_kwargs=client_kwargs,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|