前期回顾：Selenium爬取知网论文
在爬虫实战二中，我们介绍了Selenium自定义指定主题的知网论文，本次分享一个更加清晰的方法

本次爬虫知识点：Post请求构造

目标网址：老版知网

根据需要，自定义搜索条件，如下图所示

一、主要参数介绍

二、全部代码

根据以上的参数，自行更改如下代码参数。为保证全量数据，运行代码前，需要设置该搜索条件下总共的论文数。例如，搜索2020年发布在统计研究的论文共135条

import requests
from lxml import etree
import pandas as pd
import random
import numpy as np

def get_response(url,key_words,page):
    head_list = [
    "Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14",
    "Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02",
    "Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00",
    "Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00"     
    ]
    
    header = {
        'user-agent':random.choice(head_list),
        'Cookie':'UM_distinctid=17a70b84b6a105-012cb9bd046622-5771031-144000-17a70b84b6b15e; CNZZDATA1257838124=306386620-1625387566-null%7C1625387566; SID=068213; KEYWORD=%E7%BB%9F%E8%AE%A1%E4%B8%8E%E5%86%B3%E7%AD%96%24%E7%BB%9F%E8%AE%A1%E4%B8%8E%E5%B7%A8%E5%86%8C'
    }
    params={
    'searchType': 'MulityTermsSearch',
    'ParamIsNullOrEmpty': 'false',
    'Islegal': 'false',
    'Originate': key_words,
    'PublishTimeBegin': '2020年',
    'PublishTimeEnd': '2020年',
    'Order': 1,
    'Page': page,
    }
    response = requests.post(url,data=params,headers=header)
    return response.text

def parse_response(response_text):
    html = etree.HTML(response_text)
    tr_list = html.xpath('//tbody/tr')
    return  tr_list

def save_data(tr_list):
    for tr in tr_list[1:]:
        title = tr.xpath('./td[2]/div[@class="name"]/a/text()')[0]
        href = tr.xpath('./td[2]/div[@class="name"]/a/@href')[0]
        author = ','.join(tr.xpath('./td[3]//a[@class="author"]/text()'))
        source = tr.xpath('./td[4]//a/@title')[0]
        pub_ts = tr.xpath('./td[5]/div/text()')[0].strip()
        type_ = tr.xpath('./td[6]/div/text()')[0]
        cite_num = tr.xpath('./td[7]/div/text()')[0]
        article_data.append([title,href,author,source,pub_ts,type_,cite_num])


url =  'https://search.cnki.com.cn/Search/ListResult'

#这里的数字即为搜索到的总论文数
source_dict = {  '统计研究':135,
                '数理统计与管理':283,
                '统计与信息论坛':565,
                '统计与决策':3083}

article_data = []


for journal_name,all_page in source_dict.items():
    print(journal_name + '部分开始爬取')
    eve_page = int(np.ceil(all_page/20))
    for i in range(eve_page):
        try:
            r_text = get_response(url,journal_name,i)
            tr_list = parse_response(r_text)
            save_data(tr_list)
            print('第{}页开始爬取'.format(i))
        except Exception as e:
            print(e)

最终数据呈现