看之前的贴就知道我最近对股票有点兴趣，可是我对股票一窍不通怎么办呢。发现了一个网站
https://baike.baidu.com/wikitag/taglist?tagId=62991
长这样：

感觉很不错，一下子基本上涵盖了大多数的相关概念，往下翻了几下，是在太多了。一个个点不知道要看到什么时候，不如写个爬虫爬下来把。
爬文字和网页很基础，直接看代码：

import scrapy
import time

class StockTermsSpider(scrapy.Spider):
	name = 'stock_term_spider'

	def start_requests(self):

		urls = [
			'https://baike.baidu.com/wikitag/taglist?tagId=62991'
		]
		headers = {
			'Referer': 'http://quote.eastmoney.com',
			'Cookie': 'BAIDUID=DF49F703A7888AE4A13F4B2D8876DFC1:FG=1; PSTM=1532004266; BIDUPSID=8DCBE11F4D26DD0AD222967F8EE139E1; BDUSS=HI5TnV6Q1d4cHB0V2F5bXhaflFaU1NRUnN0S1Z0QTMyYnVVcndTWX5CSlhicE5iQVFBQUFBJCQAAAAAAAAAAAEAAABm0OcAMDk2MDUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFfha1tX4WtbR; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; BDSFRCVID=YyDOJeC62AMTf_QwK_QLUwtTNgg3AIbTH6aoDtn3juxV0-WB7ubtEG0Pqx8g0KubKq9vogKK0eOTHkCF_2uxOjjg8UtVJeC6EG0P3J; H_BDCLCKID_SF=tJAO_KK-tIL3fP36qR70h-4shgT22-usy65R2hcH0KLKEq62blnG5UPm0tvP-b3nBDvMohjYKfb1MRjvQbO_DPI02bDjelbGJ5TaLp5TtUJaJKnTDMRh-xPLeqjyKMnitIT9-pno0hQrh459XP68bTkA5bjZKxtq3mkjbIOFfJOKHICRe5KBDM5; H_PS_PSSID=1456_21126_29238_28519_29099_28839_29220_26350; Hm_lvt_55b574651fcae74b0a9f1cf9c8d7c93a=1561451549,1562393926,1562587230,1562588395; Hm_lpvt_55b574651fcae74b0a9f1cf9c8d7c93a=1562588395; delPer=0; PSINO=1',
			'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36',
			'Host': 'baike.baidu.com'
		}

		for url in urls:
			print("url%s" % url)
			# meta: type:0 first request 1:next request
			yield scrapy.Request(url=url, headers=headers, meta={'type': '0'}, callback=self.parse_page, dont_filter=True)

	def parse_page(self, response):
		self.log('start parse')
		# self.log(response.body)
		items = response.xpath('//*[@id="waterFall"]/div')
		i = 0

		for item in items:
			sub_url = item.xpath('a/@href').extract()[0]
			i = i + 1
			self.log(sub_url)
			time.sleep(1)
			yield scrapy.Request(url=sub_url, meta={'type': '1'}, callback=self.parse_sub_page, dont_filter=True)
		'''
		sub_url = 'https://baike.baidu.com/item/%E5%90%83%E8%B4%A7/17704185'
		yield scrapy.Request(url=sub_url, meta={'type': '1'}, callback=self.parse_sub_page, dont_filter=True)
		'''
		self.log("count = %d" % i)

	def parse_sub_page(self, reponse):
		with open('output.txt', 'a') as f:
			main_content = reponse.xpath('//div[@class = "main-content"]')
			title1 = main_content.xpath('//h1/text()').extract()[0]
			f.write(str(title1)+'\n')
			title2 = main_content.xpath('//h2/text()').extract()[0]
			f.write(str(title2)+'\n')
			paras = main_content.xpath('//div[@class = "para"]/text()')
			for para in paras:
				f.write(str(para.extract())+'\n')

	def parse(self, response):
		pass

问题来了，对于这种需要往下滚才会加载的，我们需要使用selenium模拟js脚本往下滑，
像这样：

self.driver.execute_script("document.documentElement.scrollTop=20000")

根据不同的网页结构还有不同的滑法

1、滚动条回到顶部：

js=“var q=document.getElementById(‘id’).scrollTop=0”

driver.execute_script(js）

或者

js=“var q=document.documentElement.scrollTop=0”

driver.execute_script(js)

2、滚动条拉到底部：

js=“var q=document.getElementById(‘id’).scrollTop=10000”

driver.execute_script(js)

或者

js=“var q=document.documentElement.scrollTop=10000”

driver.execute_script(js)

3、滚动条拉到指定位置（具体元素）：–最常用

target = driver.find_element_by_id(“id_keypair”)

driver.execute_script(“arguments[0].scrollIntoView();”, target)

4、通过模拟键盘DOWN（↓）来拖动：

driver.find_element_by_id(“id”).send_keys(Keys.DOWN)

百度这个比较坑爹滚一次还不够，要分开滚几次
中间件代码如下：（注意sleep的位置）

from scrapy import signals
import scrapy
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
import logging


class StockTermMiddleware(object):
    def __init__(self):
        chrome_options = Options()
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        chrome_options.add_argument('--no-sandbox')

        self.driver = webdriver.Chrome(chrome_options=chrome_options,
                                       executable_path='/Users/yaochenli/stock/stock/chromedriver')

    def process_request(self, request, spider):
        self.driver.get(request.url)
        time.sleep(3)
        if int(request.meta['type']) == 0:
            i = 0
            while i < 20:
                self.driver.execute_script("document.documentElement.scrollTop=20000")
                time.sleep(2)
                i = i+1

        html = self.driver.page_source
        return scrapy.http.HtmlResponse(request.url, body=html.encode('utf-8'), encoding='utf-8', request=request,
                                        flags=[1])

最终启用中间件就可以啦～

运行代码 scrapy crawl stock_term_spider