From 4932177f7c75da4740c82aeee86fa06c4c822b5c Mon Sep 17 00:00:00 2001 From: win Date: Sat, 21 Mar 2026 18:07:54 +0800 Subject: [PATCH] feat(01-01): create crawler_core package scaffold and pyproject.toml - Create crawler_core/pyproject.toml with setuptools build config - Add platform namespace __init__.py files for boss, qcwy, zhilian - Add requests_go==1.0.9 and tenacity>=8.0 to Pipfile [packages] - Add pytest, pytest-cov, pytest-anyio to Pipfile [dev-packages] --- Pipfile | 5 +++++ crawler_core/boss/__init__.py | 1 + crawler_core/pyproject.toml | 17 +++++++++++++++++ crawler_core/qcwy/__init__.py | 1 + crawler_core/zhilian/__init__.py | 1 + 5 files changed, 25 insertions(+) create mode 100644 crawler_core/boss/__init__.py create mode 100644 crawler_core/pyproject.toml create mode 100644 crawler_core/qcwy/__init__.py create mode 100644 crawler_core/zhilian/__init__.py diff --git a/Pipfile b/Pipfile index b7daec0..d9a5a34 100644 --- a/Pipfile +++ b/Pipfile @@ -65,12 +65,17 @@ watchfiles = "==1.0.4" websockets = "==14.1" PyExecJS = "==1.5.1" playwright = "==1.57.0" +requests_go = "==1.0.9" +tenacity = ">=8.0" asyncpg = "*" pandas = "*" openpyxl = "*" pysocks = "*" [dev-packages] +pytest = ">=8.0" +pytest-cov = ">=4.0" +pytest-anyio = "*" [requires] python_version = "3.13" diff --git a/crawler_core/boss/__init__.py b/crawler_core/boss/__init__.py new file mode 100644 index 0000000..d0b388e --- /dev/null +++ b/crawler_core/boss/__init__.py @@ -0,0 +1 @@ +"""Boss直聘 platform module.""" diff --git a/crawler_core/pyproject.toml b/crawler_core/pyproject.toml new file mode 100644 index 0000000..356cd3b --- /dev/null +++ b/crawler_core/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.backends.legacy:build" + +[project] +name = "crawler_core" +version = "0.1.0" +description = "Shared crawler core — sign algorithms, HTTP client, base classes" +requires-python = ">=3.11" +dependencies = [ + "requests_go==1.0.9", + "tenacity>=8.0", +] + +[tool.setuptools.packages.find] +where = [".."] +include = ["crawler_core*"] diff --git a/crawler_core/qcwy/__init__.py b/crawler_core/qcwy/__init__.py new file mode 100644 index 0000000..aade991 --- /dev/null +++ b/crawler_core/qcwy/__init__.py @@ -0,0 +1 @@ +"""前程无忧 (51Job) platform module.""" diff --git a/crawler_core/zhilian/__init__.py b/crawler_core/zhilian/__init__.py new file mode 100644 index 0000000..2743127 --- /dev/null +++ b/crawler_core/zhilian/__init__.py @@ -0,0 +1 @@ +"""智联招聘 platform module."""