python 爬取数据_爬取数据python

一、使用步骤

1.引入库

导入requests库和etree库

import requests
from lxml import etree
1
2

2.获取网页的HTML

html=requests.get('https://so.gushiwen.org/shiwenv_a1e7559dada7.aspx').text
#requests的get请求，直接获取网页的HTML
1
2

3.获取xpath

这个网站没有反爬，所有可以直接获取

tree=etree.HTML(html)
datas=tree.xpath('//*[@id="sonsyuanwen"]/div[1]')
#图片中的第一个箭头的位置
1
2
3

爬取的古诗

for data in datas:
    bt=data.xpath('./h1/text()')[0] #标题
    sr=data.xpath('./p/a[1]/text()')[0] #诗人
    sd=data.xpath('./p/a[2]/text()')[0] #时代
    print(bt+"\n"+sr+"\t"+sd)
    #诗句
    br=tree.xpath('//*[@id="contsona1e7559dada7"]')
    for brs in br:
        sc1=brs.xpath('./text()[1]')[0]
        sc2 = brs.xpath('./text()[2]')[0]
        sc3 = brs.xpath('./text()[3]')[0]
        sc4 = brs.xpath('./text()[4]')[0]
        print(sc1+"\n"+sc2+"\n"+sc3+"\n"+sc4)
1
2
3
4
5
6
7
8
9
10
11
12
13

4.存储为txt文档

        with open("古诗.txt","w", encoding="utf-8",newline='')as f:
            f.write(bt+"\n"+sr+"\t"+sd+sc1+"\n"+sc2+"\n"+sc3+"\n"+sc4)
1
2

总结

整体代码：

import requests
from lxml import etree

html=requests.get('https://so.gushiwen.org/shiwenv_a1e7559dada7.aspx').text

tree=etree.HTML(html)
data=tree.xpath('//*[@id="sonsyuanwen"]/div[1]')
for datas in data:
    bt=datas.xpath('./h1/text()')[0]
    sr=datas.xpath('./p/a[1]/text()')[0]
    sd=datas.xpath('./p/a[2]/text()')[0]
    print(bt+"\n"+sr+"\t"+sd)
    br=tree.xpath('//*[@id="contsona1e7559dada7"]')
    for brs in br:
        sc1=brs.xpath('./text()[1]')[0]
        sc2 = brs.xpath('./text()[2]')[0]
        sc3 = brs.xpath('./text()[3]')[0]
        sc4 = brs.xpath('./text()[4]')[0]
        print(sc1+"\n"+sc2+"\n"+sc3+"\n"+sc4)

        with open("古诗.txt","w", encoding="utf-8",newline='')as f:
            f.write(bt+"\n"+sr+"\t"+sd+sc1+"\n"+sc2+"\n"+sc3+"\n"+sc4)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22