import requests
import re
import csv  # 导入处理CSV的内置模块
import os  # 用于判断文件是否存在，避免重复写入表头

# 请求的基础网址（注意：cookie和timestamp可能过期，需定期更新）
base_url = 'https://we.51job.com/api/job/search-pc?api_key=51job&timestamp=1757825815&keyword=python&searchType=2&function=&industry=&jobArea=000000&jobArea2=&landmark=&metro=&salary=&workYear=&degree=&companyType=&companySize=&jobType=&issueDate=&sortType=0&pageNum=1&requestId=&pageSize=20&source=1&accountId=217570594&pageCode=sou%7Csou%7Csoulb&scene=7'

# 请求头（cookie需从浏览器F12开发者工具中获取最新值）
headers = {
    'referer': "https://we.51job.com/pc/search?keyword=python%E7%88%AC%E8%99%AB",
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0',
    "cookie": "partner=SEM_pcbingpz_02; guid=c6dbba77b08a532ed237a6d30d97ba51; ps=needv%3D0; 51job=cuid%3D217570594%26%7C%26cusername%3D5A%252FqMpj9ez9tY9J3CrfQpFiX42NHUDXtEKDnWGNs8os%253D%26%7C%26cpassword%3D%26%7C%26cname%3DVeJutepWmSe61aTXMh0UtQ%253D%253D%26%7C%26cemail%3D5KeCgONl9DxjAYYJtjU%252B1nQ6PZhyh640BzGmNjc983o%253D%26%7C%26cemailstatus%3D0%26%7C%26cnickname%3D%26%7C%26ccry%3D.0vuORyJTH4uQ%26%7C%26cconfirmkey%3D2806qjteGgR4Q%26%7C%26cautologin%3D1%26%7C%26cenglish%3D0%26%7C%26sex%3D0%26%7C%26cnamekey%3D28.I0RusXfUQU%26%7C%26to%3D760734587f363726a8b4b17c37ee967568c62a19%26%7C%26; sensor=createDate%3D2022-10-21%26%7C%26identityType%3D1; sajssdk_2015_cross_new_user=1; JSESSIONID=2E39E5CCD1655712E76A9E994AB5F46E; Hm_lvt_1370a11171bd6f2d9b1fe98951541941=1757829161; Hm_lpvt_1370a11171bd6f2d9b1fe98951541941=1757829161; HMACCOUNT=4D76AF8D7A0527E2; slife=lastlogindate%3D20250914%26%7C%26securetime%3DUW0EMABjBGdVPwQ4ADJbM1ZtV2Q%253D; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22c6dbba77b08a532ed237a6d30d97ba51%22%2C%22first_id%22%3A%2219946147e79b3-05826c4cb1d4988-4c657b58-1327104-19946147e7acd%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22identities%22%3A%22eyIkaWRlbnRpdHlfY29va2llX2lkIjoiMTk5NDYxNDdlNzliMy0wNTgyNmM0Y2IxZDQ5ODgtNGM2NTdiNTgtMTMyNzEwNC0xOTk0NjE0N2U3YWNkIiwiJGlkZW50aXR5X2xvZ2luX2lkIjoiYzZkYmJhNzdiMDhhNTMyZWQyMzdhNmQzMGQ5N2JhNTEifQ%3D%3D%22%2C%22history_login_id%22%3A%7B%22name%22%3A%22%24identity_login_id%22%2C%22value%22%3A%22c6dbba77b08a532ed237a6d30d97ba51%22%7D%2C%22%24device_id%22%3A%2219946147e79b3-05826c4cb1d4988-4c657b58-1327104-19946147e7acd%22%7D; acw_tc=ac11000117578296995832431e0098cbd46481cdbe5c6a1c5775e637397020; acw_sc__v2=68c65a48815b4a40eaaaf0c984f6c86d09e17a4c; ssxmod_itna=eqUO7KD5iKAItx4kDRh4Ax7uBbbG2DQq0dGMD3qiQGgDYq7=GFFDCgGz7E1jA2kh4xD0YCiwPleRqDsK5xiNDAPq0iDC8WjdoWtYu0KK5nCxx3qBcBiIpoRCOgwedoAOE0M=aNy1qZLkkhdKxoDU4GnD06qi8iD4R3Dt4DIDAYDDxDWj4DLDYoDYRT5xGpdMQnrXQRTD0YDzqDg/BmaxGfdyiY=P4DlY+AxeS45xYpdxZ07NgqDAMDi57Gdx0UaDBLKUzHDGqk6PRWihFS+H3RHsPGuDG68ZQ2NArTWPc74hT0rW8heK2NeQGeGxhjIx1GdW0tChu0wQYmQCD4iG=QYnnpZ=4DDaKa+bUGHKxSFvMUelmyYlHtRqz8gYmB4z7qRi4QqhBxxlxq4+iG0QVvPzqxCDbB5PeD; ssxmod_itna2=eqUO7KD5iKAItx4kDRh4Ax7uBbbG2DQq0dGMD3qiQGgDYq7=GFFDCgGz7E1jA2kh4xD0YCiwPlet4DWwWSCWWpT5hjyXG4hnYqWfTPD"
}

# -------------------------- 关键：CSV文件配置 --------------------------
# CSV文件路径（当前脚本所在文件夹，命名为51job_python职位.csv）
csv_file_path = "51job_python职位.csv"
# CSV表头（与后续提取的字段一一对应，顺序一致）
csv_headers = [
    "职位名称", "职位标签", "工作地区", "薪资", "公司全称",
    "职位描述", "企业类型", "所属行业", "HR姓名", "更新时间",
    "省份", "学历要求", "工作年限要求", "公司规模", "HR职位"
]


# -------------------------- 核心：数据提取与CSV写入 --------------------------
def extract_and_save_to_csv():
    # 1. 打开CSV文件：newline='' 避免空行，encoding='utf-8-sig' 支持中文显示
    with open(csv_file_path, mode='a', newline='', encoding='utf-8-sig') as f:
        # 创建CSV写入对象
        csv_writer = csv.writer(f)

        # 2. 判断文件是否为空，为空则先写入表头（避免多页爬取时重复写表头）
        if os.path.getsize(csv_file_path) == 0:
            csv_writer.writerow(csv_headers)

        # 3. 循环爬取多页（此处1-2页，可修改range(1, 11)爬10页）
        for page_num in range(1, 3):
            # 拼接当前页的URL（替换base_url中的pageNum参数）
            current_url = base_url.replace("pageNum=1", f"pageNum={page_num}")

            try:
                # 发送请求获取页面数据
                res = requests.get(url=current_url, headers=headers, timeout=10)
                res.raise_for_status()  # 若请求失败（如404/500），抛出异常
                page_text = res.text

                # 正则提取单个职位数据（匹配包含jobId的职位块）
                job_pattern = r'"jobId":".*?html"'
                job_list = re.findall(job_pattern, page_text)

                if not job_list:
                    print(f"第{page_num}页未找到职位数据，可能已爬完或链接失效")
                    continue

                # 4. 解析每个职位的字段
                for job in job_list:
                    # 提取字段：使用re.findall，若未匹配到则用默认值
                    job_name = re.findall(r'"jobName":"(.*?)"', job)[0] if re.findall(r'"jobName":"(.*?)"',
                                                                                      job) else "未知"
                    job_tags = re.findall(r'"jobTags":(.*?)jobNumString"', job)[0] if re.findall(
                        r'"jobTags":(.*?)jobNumString"', job) else "无"
                    job_area = re.findall(r'"jobAreaString":"(.*?)"', job)[0] if re.findall(r'"jobAreaString":"(.*?)"',
                                                                                            job) else "未知"
                    salary = re.findall(r'"provideSalaryString":"(.*?)"', job)[0] if re.findall(
                        r'"provideSalaryString":"(.*?)"', job) else "面议"
                    company_name = re.findall(r'"fullCompanyName":"(.*?)"', job)[0] if re.findall(
                        r'"fullCompanyName":"(.*?)"', job) else "未知"
                    job_desc = re.findall(r'"jobDescribe":"(.*?)"', job)[0] if re.findall(r'"jobDescribe":"(.*?)"',
                                                                                          job) else "无"
                    company_type = re.findall(r'"companyTypeString":"(.*?)"', job)[0] if re.findall(
                        r'"companyTypeString":"(.*?)"', job) else "未知"
                    industry = re.findall(r'"companyIndustryType1Str":"(.*?)"', job)[0] if re.findall(
                        r'"companyIndustryType1Str":"(.*?)"', job) else "未知"
                    hr_name = re.findall(r'"hrName":"(.*?)"', job)[0] if re.findall(r'"hrName":"(.*?)"', job) else "未知"
                    update_time = re.findall(r'"updateDateTime":"(.*?)"', job)[0] if re.findall(
                        r'"updateDateTime":"(.*?)"', job) else "未知"
                    province = re.findall(r'"provinceString":"(.*?)"', job)[0] if re.findall(
                        r'"provinceString":"(.*?)"', job) else "未知"
                    degree = re.findall(r'"degreeString":"(.*?)"', job)[0] if re.findall(r'"degreeString":"(.*?)"',
                                                                                         job) else "无要求"
                    work_year = re.findall(r'"workYearString":"(.*?)"', job)[0] if re.findall(
                        r'"workYearString":"(.*?)"', job) else "无要求"
                    company_size = re.findall(r'"companySizeString":"(.*?)"', job)[0] if re.findall(
                        r'"companySizeString":"(.*?)"', job) else "未知"
                    hr_position = re.findall(r'"hrPosition":"(.*?)"', job)[0] if re.findall(r'"hrPosition":"(.*?)"',
                                                                                            job) else "未知"

                    # 清理职位标签中的特殊字符（避免CSV格式错乱）
                    job_tags = re.sub(r'[\\/:"*?<>|[\]]', '', job_tags)

                    # 5. 组织当前职位的数据（顺序与CSV表头一致）
                    job_data = [
                        job_name, job_tags, job_area, salary, company_name,
                        job_desc, company_type, industry, hr_name, update_time,
                        province, degree, work_year, company_size, hr_position
                    ]

                    # 6. 写入CSV文件
                    csv_writer.writerow(job_data)
                    print(f"第{page_num}页 - 已保存：{job_name}（{company_name}）")

            except Exception as e:
                print(f"第{page_num}页爬取失败：{str(e)}")


# 执行函数
if __name__ == "__main__":
    extract_and_save_to_csv()
    print(f"\n爬取完成！CSV文件已保存至：{os.path.abspath(csv_file_path)}")