当前位置:   article > 正文

python爬虫爬取豆瓣读书Top250

python爬虫爬取豆瓣读书Top250

python爬虫爬取豆瓣读书Top250

话不多说,直接上代码!

from urllib.request import urlopen, Request
from bs4 import BeautifulSoup
import random

class DBTop250_Spider():
    url = r"http://book.douban.com/top250?start="
    page_num = 0
    top_num = 1
    headers = [
        {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:34.0) Gecko/20100101 Firefox/34.0'},
        {'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'},
        {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.12 '
                          'Safari/535.11'},
        {'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)'},
        {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:40.0) Gecko/20100101 Firefox/40.0'},
        {
            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu '
                          'Chromium/44.0.2403.89 Chrome/44.0.2403.89 Safari/537.36'}]
    file_path = "..\Files\Books"

    def Parse_Page(self):
        req = Request(self.url+str(self.page_num * 25), headers=random.choice(self.headers))
        page = urlopen(req).read()
        bsObj = BeautifulSoup(page, "html.parser")
        books_items = bsObj.find_all("a")
        for item in books_items:
            if 'title' in item.attrs:
                self.Parse_Book(item.attrs['href'])
                self.top_num += 1
        self.page_num += 1

    def Parse_Book(self, book_url):
        print("book_url is: " + book_url)
        try:
            book_request = Request(book_url, headers=random.choice(self.headers))
            book_html = urlopen(book_request).read()
            book_bsObj = BeautifulSoup(book_html, "html.parser")
            book_title = book_bsObj.h1
            book_info = book_bsObj.find("div", {"id": "info"})
            intro = book_bsObj.find_all("div", {"class": "intro"})
            if intro[0].get_text().find("展开全部") != -1:
                book_intro = intro[1]
                if intro[2].get_text().find('展开全部') != -1:
                    author_intro = intro[3]
                else:
                    author_intro = intro[2]
            else:
                book_intro = intro[0]
                if intro[1].get_text().find('展开全部') != -1:
                    author_intro = intro[2]
                else:
                    author_intro = intro[1]
            with open(self.file_path + '\Top' + str(self.top_num) + '.txt', 'w', errors='ignore') as f:
                f.write(book_title.get_text().strip() + '\n')
                info_text = book_info.get_text().split(' ')
                for info in info_text:
                    if info != '\n' and info != '':
                        f.write(info.strip())
                f.write("\n\n内容简介: ")
                f.write(book_intro.get_text())
                f.write("\n\n作者简介: ")
                f.write(author_intro.get_text())
        except Exception as e:
            if hasattr(e, "reason"):
                print("Reason: " + e.reason)

sp = DBTop250_Spider()
for i in range(1, 11):
    sp.Parse_Page()
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
  • 7
  • 8
  • 9
  • 10
  • 11
  • 12
  • 13
  • 14
  • 15
  • 16
  • 17
  • 18
  • 19
  • 20
  • 21
  • 22
  • 23
  • 24
  • 25
  • 26
  • 27
  • 28
  • 29
  • 30
  • 31
  • 32
  • 33
  • 34
  • 35
  • 36
  • 37
  • 38
  • 39
  • 40
  • 41
  • 42
  • 43
  • 44
  • 45
  • 46
  • 47
  • 48
  • 49
  • 50
  • 51
  • 52
  • 53
  • 54
  • 55
  • 56
  • 57
  • 58
  • 59
  • 60
  • 61
  • 62
  • 63
  • 64
  • 65
  • 66
  • 67
  • 68
  • 69
  • 70

运行结果:
在这里插入图片描述

声明:本文内容由网友自发贡献,不代表【wpsshop博客】立场,版权归原作者所有,本站不承担相应法律责任。如您发现有侵权的内容,请联系我们。转载请注明出处:https://www.wpsshop.cn/w/AllinToyou/article/detail/488160
推荐阅读
相关标签
  

闽ICP备14008679号