爬虫实战案例 -- 爬取豆瓣读书网页内容

1. 进入网站检查信息 , 确定请求方式以及相关数据
   

2. 找到爬取目标位置
   
   

3. 开始敲代码

# 链接网站
def url_link(url):
    res = requests.get(url,headers = headers)
    response = res.text
    parse_data(response)

# 爬取信息
def parse_data(data):
    msg = '<li\sclass="media\sclearfix">.*?'\
          '<img\sclass="subject-cover".*?src="(.*?)"/></a>.*?'\
          'class="fleft"\shref="(.*?)">(.*?)</a>.*?'\
          'class="subject-abstract\scolor-gray">(.*?)</p>'


    result = re.findall(msg,data,re.S)
	
    for i in result:
        img_url = i[0]
        bookData = i[1]
        bookName = i[2]
        bookauthor = i[3].strip()
        print('图片信息：', img_url)
        print('详情链接：', bookData)
        print('书籍名字：', bookName)
        print('作者信息：', bookauthor)
        print('======================')
        keep_data(bookName,img_url,bookData,bookauthor)

# 保存数据
def keep_data(img, data, name, author):
	# 创建文件夹
    if not os.path.exists('doubanData'):
        os.mkdir('doubanData')
	# 保存书籍信息
    with open("doubanData\db.json", "a", encoding="utf-8") as f:
        f.write("书籍名称：" + name + "\n")
        f.write("图片信息：" + img + "\n")
        f.write("书籍详情页：" + data + "\n")
        f.write('书籍作者：' + author + "\n\n")

    # 保存图片信息
    urlDta = requests.get(data).content
    with open('doubanData/{}.jpg'.format(img),'wb') as f:
        f.write(urlDta)



if __name__ == '__main__':
	# 设置爬取页数
    for i in range(1,6):
        url = f'https://book.douban.com/latest?subcat=%E5%85%A8%E9%83%A8&p={i}'
        print(f"正在爬取第{i}页")
        print("====================================================================")
        url_link(url)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

最终效果