1-2 python爬虫练习 爬取百度图片_python 使用xpath爬取百度图片

实现爬取任意数量的图片，百度图片是动态加载进来的，只用requests来爬取静态页面比较复杂，可以利用selenium和requests来爬取，其实可以单独利用selenium来爬。

需要下载chromedriver和chrome浏览器。（chromedriver和这个程序放在同一个文件夹中）
需要python下载selenuim、requests库

from selenium import webdriver
import time
from lxml import etree
import requests
import os

def mkdir(path):
    folder = os.path.exists(path)
    if not folder:
        os.makedirs(path)
        print(f"folder{path} success")
    else:
        print(f"folder{path} filed")

if __name__ == '__main__':
    word = input("关键词")
    num = eval(input("数量"))
    path = "./"+word
    mkdir(path)
    headers = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36",
        "referer": "https://image.baidu.com/"
    }
    #打开浏览器驱动
    bro = webdriver.Chrome(executable_path='./chromedriver.exe')
    #请求浏览器
    bro.get("https://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=index&fr=&hs=0&xthttps=111110&sf=1&fmq=&pv=&ic=0&nc=1&z=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&word="+word)
    index = 1
    #获取当前状态页面源代码
    page = bro.page_source
    tree = etree.HTML(page)		#解析
    top = boxes = tree.xpath('//div[@class="imgbox"]')
    #第一次执行，先把有的这一部分弄出来

    for box in boxes:
        url = box.xpath('./a/img/@src')[0]
        print(index,url)
        content = requests.get(url=url,headers=headers).content
        with open(file=path+f'/{index}.jpg',mode="wb") as fp:
            fp.write(content)
        index += 1
    while(index<num):

        #让浏览器滚轮向下滑动        
        bro.execute_script("window.scrollTo(0,document.body.scrollHeight)")
        #继续解析
        page = bro.page_source
        tree = etree.HTML(page)
        boxes = tree.xpath('//div[@class="imgbox"]')
        #如果现在解析到的和上一次不一样，那么找到多的这一部分打印出来，并且这一次作为上一次
        if(top!=boxes):
            more = boxes[len(top)-1:-1]
            for a in more:
            	#获取图片url
                url = a.xpath('./a/img/@src')[0]
                #图片二进制内容
                content = requests.get(url=url,headers=headers).content
                with open(file=path+f'/{index}.jpg',mode="wb") as fp:
                    fp.write(content)
                print(index,url)
                index += 1
            top = boxes
    #关闭浏览器
    bro.close()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64