爬取笔趣看小说

本文来源吾爱破解论坛

之前看过一位老哥的帖子，想学习学习，就在那位老哥的代码上进行修改，换了一个网站，减小了一点需求。模仿了一下，发现最后生成txt格式的文本，排版有点问题，我试了一下，发现解决不了，放弃了。哪位大佬知道原因，指导一波，感谢。[Python] 纯文本查看 复制代码

import os, requests, re
from time import sleep
from bs4 import BeautifulSoup
from random import uniform


# 网址解析
def url_open(url):
    headers = {}
    headers[
        "User-Agent"] = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36"
    response = requests.get(url, headers=headers)
    response.encoding = "gbk"  # 该网站的编码格式：utf-8
    html = response.text
    return html
def replaceNovel(content):
    #用来进行页面排版，最后还是不太理想，不知道为什么没有向左对齐，第一行移动，下一行也移动
    temp=content.replace('app2();','',2)
    mode=r'\([url]https://www.biqukan.com.[/url]+wap.biqukan.com'
    src=re.sub(mode,'',temp,flags=re.S)
    final=src.replace('。　　','。\n    ',50)
    return final
# 小说内容获取与保存
def get_and_save_data(novel_url):
    chapters_url_all=[];
    html_1 = url_open(novel_url)
    soup_1 = BeautifulSoup(html_1, "html.parser")
    nodes=soup_1.dl.find_all("dd")
    novel_name=soup_1.find(class_="info").h2.text;
    print(novel_name)
    flag = False
    prefix=re.findall(r'(.+)m/',novel_url)[0]
    prefix=prefix+"m"
    for node in nodes:
        url=node.a["href"]
        text=node.a.text
        if text[0:3]=='第一章':
            flag=True
        if flag is True:
            chapters_url_all.append(prefix+url)
    #print(chapters_url_all)
    # 逐页打开小说章节网址并获取内容保存
    for each in chapters_url_all:
        chapters_url =each
        html = url_open(chapters_url)

        soup = BeautifulSoup(html, "html.parser")
        chapters_name = soup.find(class_="content").h1.text  # 抓取章节名称
        print("正在下载《%s》:%s" % (novel_name, chapters_name))

        # 小说内容抓取
        content = soup.find("div", id="content").text
        contents=replaceNovel(content)
        #print(contents)
        with open("%s.txt" % novel_name, "a", encoding="utf-8") as g:
            g.write("\n" * 3 + "                               " + chapters_name + str("\n") * 3)
            #g.write("    " + contents)
            g.write(contents)
        #break
        slee_time = uniform(0.35, 0.75)
        sleep(slee_time)

    print("小说%s已下载完毕" % novel_name)
    sleep(2)

def main():
    # 设置工作路径
    path = r'd:\Pythontest'
    if os.getcwd() != path:
        if os.path.exists(path) == False:
            os.mkdir(path)
            os.chdir(path)
        else:
            os.chdir(path)
    #这个地址取你要爬的小说地址，我在站内搜索结果查不到，只能通过去百度搜小说名，之后进入页面
    root_url = "https://www.biqukan.com/73_74917/"
    get_and_save_data(root_url)


if __name__ == "__main__":
    main()

格式.JPG (15.9 KB, 下载次数: 1)

下载附件  保存到相册

2020-1-19 15:34 上传