聊城网站建设费用,wordpress 查看菜单,重庆双福建设开发有限公司网站,网站开发图书管理系统案例需求#xff1a;
1.使用selenium自动化爬虫爬取哔哩哔哩排行榜中舞蹈类的数据#xff08;包括视频标题、up主、播放量和评论量#xff09;
2.利用bs4进行数据解析和提取
3.将爬取的数据保存在本地json文件中
4.保存在excel文件中
分析#xff1a;
1.请求url地址
1.使用selenium自动化爬虫爬取哔哩哔哩排行榜中舞蹈类的数据包括视频标题、up主、播放量和评论量
2.利用bs4进行数据解析和提取
3.将爬取的数据保存在本地json文件中
4.保存在excel文件中
分析
1.请求url地址https://www.bilibili.com/v/popular/rank/dance 2.加载等待事件否则获取数据不充分 wait WebDriverWait(self.browsers, 280)
wait.until(EC.presence_of_element_located((By.CLASS_NAME, rank-item)))
time.sleep(5) 3.获取相应内容 last_height self.browsers.execute_script(return document.body.scrollHeight)
while True:self.browsers.execute_script(window.scrollTo(0, document.body.scrollHeight);)time.sleep(5)data self.browsers.page_source # 获取网页源码self.parse_data(datadata)new_height self.browsers.execute_script(return document.body.scrollHeight)if new_height last_height:breaklast_height new_height 4.使用bs4解析数据 soup BeautifulSoup(data, lxml)
titles soup.select(.info .title) # 标题
up_names soup.select(.info .up-name) # up主
# :nth-of-type(2) 用于选择指定类型的第二个元素
play_counts soup.select(.info .detail-state .data-box:nth-of-type(1)) # 播放量
comment_counts soup.select(.info .detail-state .data-box:nth-of-type(2)) # 评论量
rank_data {}
print(len(titles))
for title, name, play_count, comment_count in zip(titles, up_names, play_counts, comment_counts):t title.get_text().strip()n name.get_text().strip()p play_count.get_text().strip()c comment_count.get_text().strip()print(标题, t)print(up主, n)print(播放量, p)print(评论量, c)print()5.保存在本地json文件中 with open(rank_data.json, a, encodingutf-8) as f:f.write(json.dumps(rank_data, ensure_asciiFalse) \n) 6.保存在excel文件中 wb workbook.Workbook()#创建一个EXcel对象 就相当于是要生成一个excel 程序
ws wb.active #激活当前表
ws.append([标题,up主,播放量,评论量]) #保存数据
def save_data(self,title,name,paly,comment):ws.append([title,name,paly,comment])# 保存为Excel数据wb.save(哔哩哔哩排行榜数据.xlsx) 案例代码
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
from openpyxl import workbook #第三方模块 需要安装
import time
import jsonwb workbook.Workbook()#创建一个EXcel对象 就相当于是要生成一个excel 程序
ws wb.active #激活当前表
ws.append([标题,up主,播放量,评论量])class Spider:def __init__(self):self.url https://www.bilibili.com/v/popular/rank/danceself.options webdriver.ChromeOptions()self.options.add_experimental_option(excludeSwitches, [enable-automation])self.browsers webdriver.Chrome(optionsself.options)# 访问哔哩哔哩排行榜def get_bili(self):self.browsers.get(self.url)wait WebDriverWait(self.browsers, 280)wait.until(EC.presence_of_element_located((By.CLASS_NAME, rank-item)))time.sleep(5)# 获取响应内容def get_data(self):last_height self.browsers.execute_script(return document.body.scrollHeight)while True:self.browsers.execute_script(window.scrollTo(0, document.body.scrollHeight);)time.sleep(5)data self.browsers.page_source # 获取网页源码self.parse_data(datadata)new_height self.browsers.execute_script(return document.body.scrollHeight)if new_height last_height:breaklast_height new_height# 解析信息def parse_data(self, data):soup BeautifulSoup(data, lxml)titles soup.select(.info .title) # 标题up_names soup.select(.info .up-name) # up主# :nth-of-type(2) 用于选择指定类型的第二个元素play_counts soup.select(.info .detail-state .data-box:nth-of-type(1)) # 播放量comment_counts soup.select(.info .detail-state .data-box:nth-of-type(2)) # 评论量rank_data {}print(len(titles))for title, name, play_count, comment_count in zip(titles, up_names, play_counts, comment_counts):t title.get_text().strip()n name.get_text().strip()p play_count.get_text().strip()c comment_count.get_text().strip()print(标题, t)print(up主, n)print(播放量, p)print(评论量, c)print()self.save_data(t,n,p,c)rank_data[标题] trank_data[up主] nrank_data[播放量] prank_data[评论量] cwith open(rank_data.json, a, encodingutf-8) as f:f.write(json.dumps(rank_data, ensure_asciiFalse) \n)#保存数据def save_data(self,title,name,paly,comment):ws.append([title,name,paly,comment])# 保存为Excel数据wb.save(哔哩哔哩排行榜数据.xlsx)if __name__ __main__:s Spider()s.get_bili()s.get_data()
运行结果
