selenium 获取网站 全部 详细信息页面的 链接

导库

#!/usr/bin/env Python3
# -*- coding: utf-8 -*-
# @Software: PyCharm
# @virtualenv：workon
# @contact: contact information
# @Desc：Code descripton
__author__ = '未昔/AngelFate'
__date__ = '2019/8/28 21:05'
import headers_ # 随机的headers

from selenium import webdriver

加载启动项

 # 加载启动项
    def __init__(self,start_url):
        self.start_url = start_url

        self.option = webdriver.ChromeOptions()
        self.option.add_argument('headless')  
        self.option.add_argument('user-agent="%s"' % headers_.getCon_(), )  
        print(headers_.getCon_())

        self.driver = webdriver.Chrome(options=self.option)
        self.driver.get(self.start_url)
        self.driver.maximize_window()  # 窗口最大化

        self.jobUrlList=[] # 存放所有的子页面的链接

        self.get_spiderJobCon() # 调取选择抓取内容的函数

自定义抓取内容

job_name = input('请输入提取的信息：')
print(job_name)

try:  # 输入
	self.driver.find_element_by_xpath('*//input[@name="key"]').send_keys(job_name)
except Exception as e:
	print(e,'输入信息失败！')

try:
	self.driver.find_elements_by_css_selector(".search-btn.float-right")[0].click()
except Exception as e:
	print(e, '查询按钮点击失败！')

获取本页所有的子页面链接和下一页

  print('当前页面: ',self.driver.current_url[-2],self.driver.current_url[-1])

  # 获取本页所有url的信息
  node_list = self.driver.find_elements_by_css_selector(".sojob-item-main.clearfix")  签

  for node in node_list:
	  job_url = node.find_element_by_xpath('*//div[@class="job-info"]/h3/a').get_attribute('href')
	  self.jobUrlList.append(job_url)

  next_page_url = self.driver.find_element_by_link_text('下一页').get_attribute('href')# 获取下一页
  return self.getAllCon(next_page_url)

提取所有页面数据

while True:
  try:
	if next_page_url:
		# print('下一页链接: ',next_page_url)
		self.driver.get(next_page_url)
		self.get_NeedCon()
	else:
		self.driver.quit()  # 关闭浏览器
		break

	print(self.jobUrlList)
	for url in self.jobUrlList:
		with open('所有子页面链接链接.txt','a',encoding='utf8') as fn:
			fn.write(url)
			fn.write('\n')
  except Exception as e:
	  print(e,'点击下一页失败')
	  self.driver.quit() # 关闭浏览器
	  break