Python 网络爬虫进阶

1. 确定 URL 并抓取页面代码，添加headers

from urllib import request
import re
 
# 定义url
page = 100
url = 'https://tieba.baidu.com/f?kw=%E5%BC%B1%E6%99%BA&ie=utf-8&pn='+str(page)
try:
  headers ={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36'}
    # 定义请求，传入请求头
    req = request.Request(url,headers=headers)
    # 打开网页
    resp = request.urlopen(req)
    # 打印响应,解码
    # print(resp.read().decode('utf-8'))
    content = resp.read().decode('utf-8')
    print(content)
 
except request.URLError as e:
    # 打印响应码
    if hasattr(e,'code'):
        print(e.code)
    # 打印异常原因
    if hasattr(e,'reason'):
        print(e.reason)

2.查看爬取的内容，并匹配正则表达式