# 导入程序所需要的库。
import requests
from bs4 import BeautifulSoup
import time
# 加入请求头伪装成浏览器
headers = {
#通过Chrome浏览器复制User-Agent
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
}
# 定义判断用户性别的函数
def judgment_sex(class_name):
if class_name == ['member_ico1']:
return '女'
else:
return '男'
# 获取详细页URL函数
def get_links(url):
try:
wb_date = requests.get(url, headers)
except ConnectionAbortedError:
print('拒绝连接')
soup = BeautifulSoup(wb_date.text, 'lxml')
links = soup.select('#page_list > ul > li > a')
for link in links:
herf = link.get("href")
get_info(herf)
# 获取网页信息函数
def get_info(url):
wb_date = requests.get(url, headers)
soup = BeautifulSoup(wb_date.text, 'lxml')
#通过浏览器copy selector
tittles = soup.select('div.pho_info > h4')
addresses = soup.select('span.pr5')
prises = soup.select('#pricePart > div.day_l > span')
images = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > a > img')
names = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')
sexs = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > div')
for tittle, address, prise, image, name, sex in zip(tittles, addresses, prises, images, names, sexs):
date = {
'tittle': tittle.get_text().strip(),
'address': address.get_text().strip(),
'price': prise.get_text(),
'image': image.get("src"),
'name': name.get_text(),
'sex': judgment_sex(sex.get("class"))
}
print(date)
if __name__ == '__main__':
urls = ['http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format(number) for number in range(1, 14)]
for single_url in urls:
get_links(single_url)
#休眠十秒，防止被封IP
time.sleep(10)
#缺点：缺少IP管理，采用休眠方法，效率低
内容来源于网络如有侵权请私信删除
- 还没有人评论，欢迎说说您的想法！
客服
