Python爬虫——xpath解析实例_使用xpath爬取三国演义

xpath解析

• 实例化etree对象
• 调用etree对象中的xpath方法
  

一、xpath爬取三国演义标题

import requests
from lxml import etree

if __name__ == '__main__':
    headers={
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0'
    }
    url = 'https://www.gushicimingju.com/novel/sanguoyanyi/'
    page_text = requests.get(url=url, headers=headers).text
    tree = etree.HTML(page_text)
    # 找到对应的标签
    li_list = tree.xpath('//div[@class="main-content"]/ul/li')
    fp = open('三国演义章节标题.txt', 'w', encoding='utf-8')
    for li in li_list:
        title = li.xpath('./a/text()')[0]
        fp.write(title + '\n')
        print(title)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

二、爬取网页图片

• 注意：爬取得到图片名称后，需要解决乱码问题
  img_name = img_name.encode(‘iso-8859-1’)
  img_name = img_name.decode(‘gbk’)

import requests
import os
from idna import unicode
from lxml import etree

if __name__=='__main__':
    url = 'https://pic.netbian.com/4kdongwu/'

    headers={
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0'
    }
    response = requests.get(url=url, headers=headers)
    page_text = response.text
    tree = etree.HTML(page_text)
    li_list = tree.xpath('//div[@class="slist"]/ul/li')
    print(li_list)

    if not os.path.exists('pictures'):
        os.mkdir('pictures')

    for li in li_list:
        img_src = 'https://pic.netbian.com/' + li.xpath('./a/img/@src')[0]
        img_name = li.xpath('./a/img/@alt')[0] + '.jpg'
        img_name = img_name.encode('iso-8859-1')
        img_name = img_name.decode('gbk')
        print(img_name )
        # print(img_src)
        img_data = requests.get(url=img_src , headers=headers).content
        img_path = 'pictures/' + img_name
        with open(img_path,'wb') as fp:
            fp.write(img_data)

    print('over!!!')

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

三、爬取城市的名字

from lxml import etree
import requests

if __name__=='__main__':
    url = 'http://www.aqistudy.cn/historydata/'
    headers={
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0'
    }
    response = requests.get(url=url, headers=headers)
    page_text = response.text
    tree = etree.HTML(page_text)
    # 存储所有城市名称
    city = []
    # 爬取热门城市
    hot_city_li = tree.xpath('//div[@class="bottom"]/ul/li')
    for li in hot_city_li:
        hot_city = li.xpath('./a/text()')[0]
        print(hot_city)
        city.append(hot_city)
    # 非热门城市
    nohot_city_li = tree.xpath('//ul[@class="unstyled"]/div/li')
    for li in nohot_city_li:
        nohot_city = li.xpath('./a/text()')[0]
        print(nohot_city)
        city.append(nohot_city)

    print('-----------------')
    print(city)
    print(len(city))
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29