如何在Python中实现一个简单的爬虫程序

1

2

3

4

5

url = "目标网站的URL"

response = requests.get(url)

html = response.text

soup = BeautifulSoup(html, "html.parser")

1

2

3

4

5

6

7

# 示例：提取新闻标题和链接

news_list = soup.find_all("a", class_="news-title")  # 假设新闻标题使用CSS类名 "news-title"

for news in news_list:

    title = news.text

    link = news["href"]

    print(title, link)

1

2

3

4

5

6

7

# 示例：将数据存储到文件

with open("news.txt", "w", encoding="utf-8") as f:

    for news in news_list:

        title = news.text

        link = news["href"]

        f.write(f"{title}    {link}

")

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

import time

# 示例：设置延时和爬取数量

interval = 2  # 延时2秒

count = 0  # 爬取数量计数器

for news in news_list:

    if count < 10:  # 爬取10条新闻

        title = news.text

        link = news["href"]

        print(title, link)

        count += 1

        time.sleep(interval)  # 延时

    else:

        break