家居网站建设咨询,站外推广免费网站,网站推广服务网站连锁,做门户网站用什么系统之前看到有人写糗事百科的爬虫#xff0c;就爬了几个页面#xff0c;感觉太少#xff0c;一个专业的段子手怎么能忍#xff1b; 本文中使用多进程加多线程#xff0c;段子用户id保存至redis数据库#xff0c;用户数据及段子内容存储至mongodb#xff1b; 本人自己的代…之前看到有人写糗事百科的爬虫就爬了几个页面感觉太少一个专业的段子手怎么能忍 本文中使用多进程加多线程段子用户id保存至redis数据库用户数据及段子内容存储至mongodb 本人自己的代理池前段时间没了这里用的是阿布云代理说的是每秒支持并行5个代理其实没有这么多买了三个账号连续爬一天总共爬到30多万个用户数据段子200多万个 阿布云账号一小时一块钱 数据库 段子 这是整个爬取的程序缩略图 程序大概的结构是
用户id程序开始打开糗事百科历史网页从中爬取用户id放入redis正常爬取的时候保存本用户关注人放入redis同时也会按照一定概率打开糗事百科历史网页保存用户id用户内容从redis数据库0中取出一个用户id爬取内容爬取成功则将用户id保存至redis数据库1中 爬取时会首先读取当前用户总共段子的页面数之后依次爬取保存的内容包括网页上可以看到的所有用户信息及相关段子程序中有单进程与多进程的选择如果只买一个账号的话还是用单进程吧如果自己有代理池那就可以随意玩了
程序已放至本人GitHub
import requests
from bs4 import BeautifulSoup
import myconfig
import time
import pymysql
import multiprocessing
import redis
import random
import pymongo as mo
import re
import threading
import sysclass qsbk():糗事百科段子爬虫爬取用户id放入redis读取id并将用户信息及笑话保存至mongodb# 使用阿布云代理访问def proxy(self): # 获取代理if random.random() 0:conn pymysql.connect(myconfig.host, myconfig.user, myconfig.passwd, myconfig.DB_NAME)cursor conn.cursor()cursor.execute(select * from ipproxy.httpbin;)# procursor.fetchone()count_all cursor.fetchall()cursor.close()conn.close()ip random.choice(count_all)ip ip[1] : str(ip[2])proxies {http: http:// ip, https: https:// ip}else:proxyHost http-dyn.abuyun.comproxyPort 9020# 代理隧道验证信息dd random.random()# if dd 0.333:# proxyUser H8X7661D3289V75D# proxyPass C5EC2166093B3548# elif dd 0.6666:# proxyUser H746QK9967YC612D# proxyPass 541E8B324C476D54# else:# proxyUser H184S812T5JOWA3D# proxyPass 04410CA8089EF4CCproxyUser H887SMOL77Q0848DproxyPass FD75684EF149F5D1proxyMeta http://%(user)s:%(pass)s%(host)s:%(port)s % {host: proxyHost,port: proxyPort,user: proxyUser,pass: proxyPass,}proxies {http: proxyMeta,https: proxyMeta,}return proxies# 读取页面所有页面都在这里读取def getBSurl(self, url):# proxy 115.202.190.177:9020# proxies {http: http:// proxy,# https: https:// proxy}kk 1while True:try:r2 requests.get(url, headersmyconfig.headers(), proxiesself.proxy(), timeoutmyconfig.timeout) #rc2 BeautifulSoup(r2.content, lxml)if rc2.text.find(糗事百科验证服务) 0:print(这个ip被封了)else:breakexcept Exception as e:print(qqqqqqqq{}qqqqqqqq.format(repr(e)))time.sleep(0.1)kk kk 1if kk 100:print(url)print(连接好多次都连不上)sys.exit(1)return rc2# 此人页面个数def article_page(self, rc2):aa rc2.select(ul[classuser-navcnt])[0].select(a)[-2].textreturn int(aa)# 获取人物属性def people_attre(self, rc2):rc3 rc2.select(div[classuser-statis user-block])try:pic rc2.select(div[classuser-header])[0].select(img)[0].attrs[src]except:print(rc2)name rc2.select_one(div[classuser-header-cover]).text.strip(\n)content1 rc3[0]funs_num content1.select(li)[0].text.split(:)[1]atten_num content1.select(li)[1].text.split(:)[1]qiushi_num content1.select(li)[2].text.split(:)[1]comment_num content1.select(li)[3].text.split(:)[1]face_num content1.select(li)[4].text.split(:)[1]choice_num content1.select(li)[5].text.split(:)[1]content2 rc3[1]marri content2.select(li)[0].text.split(:)[1]horoscope content2.select(li)[1].text.split(:)[1]job content2.select(li)[2].text.split(:)[1]hometown content2.select(li)[3].text.split(:)[1]total_time content2.select(li)[4].text.split(:)[1]people_att {name: name, pic: pic, funs_num: funs_num, atten_num: atten_num, qiushi_num: qiushi_num,comment_num: comment_num, face_num: face_num, choice_num: choice_num, marri: marri,horoscope: horoscope, job: job, hometown: hometown, total_time: total_time}return people_att# 获取糗事内容及地址def article_site(self, rc2):aa rc2.find_all(idre.compile(article))bodyout {}for a in aa:try:pic a.find(srcre.compile(//pic)).attrs[src]except:pic 0site a.select_one(li[classuser-article-text] a).get(href).split(/)[2] # 网址body a.select_one(li[classuser-article-text] a).text.strip(\n) # 内容bb re.findall(r\d\.?\d*, a.select_one(li[classuser-article-stat]).text) # 评论smile bb[0]comment bb[1]date bb[2] bb[3] bb[4]bodyout[site] {smile: smile, comment: comment, date: date, body: body, pic: pic}# bodyout.append([site, smile, comment, date, body])return bodyout# 获取文章评论的人并保存至redisdef get_people(self, rc2):aa [x.find(hrefre.compile(/users/)).get(href).split(/)[2] for x inrc2.select(li[classuser-article-vote])]for a in aa:self.save_red(a)# 获取随机历史段子的人加入redisdef addpeople(self):url https://www.qiushibaike.com/history/rc2 self.getBSurl(url)for a in rc2.select(a[href*/users/]):b a.get(href).strip(/).split(/)[1]try:int(b)if len(b) 7 or len(b) 8 or len(b) 6:self.save_red(b)except:pass# 获取关注人写入redisdef get_follows(self, begin_people):# returnurl https://www.qiushibaike.com/users/ begin_people /follows/rc2 self.getBSurl(url)for a in rc2.select(a[href*/users/]):b a.get(href).strip(/).split(/)[1]try:int(b)if len(b) 7 or len(b) 8:self.save_red(b)except:pass# 将筛选到的人保存至redisdef save_red(self, a):returntry:red0 redis.StrictRedis(host127.0.0.1, port6379, db0)red1 redis.StrictRedis(host127.0.0.1, port6379, db1)if red0.keys(a) [] and red1.keys(a) []:red0.lpush(a, 0)print(给库新加了一个)# return aaexcept Exception as e:print(repr(e))print(Redis Connect Error!)sys.exit(1)# 将爬到的人所有内容保存至mongodbdef save_mo(self, savedata):try:client mo.MongoClient(localhost, 27017)databases_name qsbk2tablename qsbk2db client[databases_name][tablename]db.save(savedata)client.close()except Exception as e:print(repr(e))print(Mongodb Connect Error!)sys.exit(1)# 获取时间def get_time(self, start, end):a float(format(end - start, 0.2f))return a# 开始的人def begin_people(self):red0 redis.StrictRedis(host127.0.0.1, port6379, db0)yield red0.randomkey().decode()# 主循环def get_all(self, begin_people):url https://www.qiushibaike.com/users/ begin_people /articles/# print(url)rc2 self.getBSurl(url)self.get_follows(begin_people)try:if 当前用户已关闭糗百个人动态 in rc2.select_one(div[classuser-block user-setting clearfix]).text:people_att {}peopname rc2.select_one(div[classuser-header-cover]).text.strip(\n)people_att[flag] 2people_att[_id] begin_peoplepeople_att[name] peopnamereturn 1except:passtry:rc2.select_one(div[classuser-header-cover]).text.strip(\n)except:print({}这个人空间没了.format(url))print(rc2)return 1people_att self.people_attre(rc2) # 个人属性people_att[_id] begin_peopleif rc2.select_one(div[classuser-block user-article]) None:people_att[flag] 1 # 这个人没有糗事print({}这个糗事少.format(url))self.save_mo(people_att)return 1qs self.article_site(rc2) # 第一页的段子self.get_people(rc2) # 把评论的人加入列表allpage self.article_page(rc2)pageout 1for i in range(allpage - 1): # 从第二页开始page i 2pageout pageout 1url https://www.qiushibaike.com/users/ begin_people /articles/page/ str(page) /rc2 self.getBSurl(url)qs dict(qs, **self.article_site(rc2))if len(self.article_site(rc2)) 1:break# print(page)# print(len(article_site(rc2)))self.get_people(rc2)people_att[flag] 1self.save_mo(dict(people_att, **qs))print({}成功保存{}个页面.format(url, pageout))return 1# 多进程入口
def mulpro(peop):q qsbk()while True:peop next(q.begin_people())if q.get_all(peop) 1:red0.move(peop, 1)if random.random() 0.1:q.addpeople()else:pass# 多线程入口
def multhread(n):threads []q qsbk()for t in range(n):threads.append(threading.Thread(targetmulpro, args(next(q.begin_people()),)))for t in threads:# t.setDaemon(True)t.start()for t in threads:t.join()if __name__ __main__:crqsbk qsbk()crqsbk.addpeople()red0 redis.StrictRedis(host127.0.0.1, port6379, db0)flag 1 # 1单进程单线程2多进程3多线程4多进程多线程if flag 1: # 单进程单线程while True:peop next(crqsbk.begin_people())if crqsbk.get_all(peop) 1:red0.move(peop, 1)if random.random() 0.000001:crqsbk.addpeople()else:pass# red0.lpush(manlist, begin_people)# begin_people begin_people()# time.sleep(2)elif flag 2: # 多进程numList []for i in range(12):p multiprocessing.Process(targetmulpro, args(next(crqsbk.begin_people()),))numList.append(p)p.start()elif flag 3: # 多线程threads []for t in range(8):threads.append(threading.Thread(targetmulpro, args(next(crqsbk.begin_people()),)))for t in threads:# t.setDaemon(True)t.start()for t in threads:t.join()elif flag 4: # 多进程多线程numList []for i in range(8):p multiprocessing.Process(targetmulthread, args(2,))numList.append(p)p.start()print(finish)
