利用Fiddler抓包和py的requests库爬取QQ空间说说内容并写入文件

本文来源吾爱破解论坛

Fiddler 抓取页面截图

tttttttttttttttt.jpg (158.74 KB, 下载次数: 1)

下载附件  保存到相册

2019-9-11 18:05 上传

#!C:\Program Files\Python36 python
# -*- coding: UTF-8 -*-
"""
@author: 东方不败
QQ空间说说爬取程序
1. 登录QQ空间，获取cookie
2. 利用Fiddler抓取QQ空间说说的URL(Fiddler显示response直接解析成Json，但是自己尝试并未成功，带研究）
3. 利用requests库爬取页面数据
4. 利用re匹配页面内容，获取说说的createTime和content
5. 文件保存写入txt文件
"""


class Spider:
    """emotionSpider类"""
    def __init__(self, cookie):
        """初始化"""
        self.page = 0       # 记录爬取页数
        self.counts = 0     # 记录爬取条数
        self.url = "https://user.qzone.com"      # 爬取的url
        self.cookie = cookie    # 登录cookie

    def get(self, url):
        """利用requests爬取页面内容，并转化成str返回"""
        import requests     # 导入requests库

        """组合headers"""
        headers = {
            "Cookie": self.cookie,
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
        }

        self.url = url

        # 特别提醒：verify=False参数必不可少，关闭HTTPS验证，否则可能报错(踩过的坑)
        try:
            response = requests.get(self.url, headers=headers, verify=False).content
            html = str(response, 'utf-8')
            return html
        except Exception as err:
            print("Error:" + err)

    def analyse(self, content):
        """利用re匹配createTime和con"""
        import re
        con = re.findall("\"con\":\"([^\"]+)", content)
        createTime = re.findall("\"createTime\":\"([^\"]+)", content)
        emotion = tuple(zip(createTime, con))
        return emotion

    def saveEmotin(self, emotion):
        """保存emotion到TXT文件"""
        for i in emotion:
            # 参数encoding='utf-8'要带，避免Unicode编码解析保存(踩过的坑)
            try:
                with open("emotion.txt", "a+", encoding='utf-8') as f:
                    f.write(i[0])
                    f.write(":")
                    f.write(i[1])
                    f.write("\n")
                    self.counts += 1
                    print("已经保存 "+str(self.counts)+" 条说说")
            except Exception as err:
                print("Error:"+err)
        self.page += 1
        print("第"+str(self.page)+"页说说爬取完毕")
        print("#"*50)


if __name__ == "__main__":
    pos = 0
    url = ":"   # Fiddler 抓取的url
    cookie = ":"  # cookie
    spider = Spider(cookie)
    while True:
        emotion = spider.analyse(spider.get(url))
        if len(emotion) > 0:
            spider.saveEmotin(emotion)
            pos += 20   # 通过pos控制下一页
            url = "https://user.qzone.qq.com/proxy/domain/taotao.qq.com/cgi-bin/emotion_cgi_msglist_v6?uin=65&ftype=0&sort=0&pos="+str(pos)+"&num=20"
        else:
            print("+"*50)
            print("爬取结束!!!共计爬取"+str(spider.counts)+"条说说")
            break

rrrrrrrrrrrrrrrrr.jpg (228.89 KB, 下载次数: 1)

下载附件  保存到相册

2019-9-11 18:11 上传