1 from lxml import etree
2 from selenium import webdriver
3 import pymysql
4
5 def Geturl(fullurl):#获取每个招聘网页的链接
6 browser.get(fullurl)
7 shouye_html_text = browser.page_source
8 shouye_ele = etree.HTML(shouye_html_text)
9 zf_list = shouye_ele.xpath('/html/body/div[4]/div[1]/div[2]/ul/li/div/h3/a/@href')#链接url
10 zf_url_list = []
11 for zf_url_lost in zf_list:
12 zf_url = 'https://bj.5i5j.com'+zf_url_lost
13 zf_url_list.append(zf_url)
14 return zf_url_list
15 def Getinfo(zp_url_list):
16 for zp_url in zp_url_list:
17 browser.get(zp_url)
18 zp_info_html = browser.page_source
19 zp_ele = etree.HTML(zp_info_html)
20 zp_info_title = str(zp_ele.xpath('//html/body/div[3]/div[1]/div[1]/h1/text()')[0])
21 zp_info_num = str(zp_ele.xpath('/html/body/div[3]/div[2]/div[2]/div[1]/div[1]/div/p[1]/text()')[0])+'元/月'#价格
22 zp_info_type = str(zp_ele.xpath('/html/body/div[3]/div[2]/div[2]/div[1]/div[2]/div/p[1]/text()')[0])#户型
23 zp_info_zone = str(zp_ele.xpath('/html/body/div[3]/div[2]/div[2]/div[1]/div[3]/div/p[1]/text()')[0])+'平米'#房屋大小
24 zp_info_need_1 = str(zp_ele.xpath('/html/body/div[3]/div[2]/div[2]/div[2]/ul/li[1]/span/text()')[0])#房屋信息
25 zp_info_need_2 = str(zp_ele.xpath('/html/body/div[3]/div[2]/div[2]/div[2]/ul/li[1]/a/text()')[0])#房屋信息
26 zp_info_need = zp_info_need_1+zp_info_need_2
27 connection = pymysql.connect(host='localhost', user='root', password='1234', db='5i5j', )
28 try:
29 with connection.cursor() as cursor:
30 sql = "INSERT INTO `5i5j_info` (`title`,`num`,`type`, `zone`,`need`) VALUES (%s,%s,%s,%s, %s)"
31 cursor.execute(sql, (zp_info_title,zp_info_num,zp_info_type,zp_info_zone,zp_info_need))
32 connection.commit()
33 finally:
34 connection.close()
35 print(zp_info_title,zp_info_num,zp_info_type,zp_info_zone,zp_info_need)
36 if __name__ == '__main__':
37 browser = webdriver.Chrome()
38 pags = int(input('需要几页?'))
39 for i in range(1,pags+1):
40 url = 'https://bj.5i5j.com/zufang/huilongguan/n{}/'
41 fullurl = url.format(str(i))
42 zf_url_list = Geturl(fullurl)
43 print(fullurl)
44 # print(zf_url_list)
45 Getinfo(zf_url_list)
46 browser.close()
