Python新手入门爬虫练习——天天基金_怎么爬取天天基金网的基金数据

利用Python爬取天天基金的数据。
先打开天天基金网，然后选择感兴趣的主题基金，这里以煤炭为例。
进入页面之后，复制一个基金名称，在浏览器的控制台Network搜索这个名称。
点击我们搜索到的js，切换到Headers，这里有几个信息是我们需要的：General里的Request URL、Request Headers里的Cookie、Host、Referer、User-Agent。

import pandas as pd
import requests
import re
import json
import time
import math

#定义一个fund_obj用于存放，刚才在浏览器上获取到的几个值
fund_obj = {
    'isbuy': 1,
    'sort': 'TRY',
    'sorttype': 'desc',
    'callback': 'jQuery18306680153539612363_1681216767557',
    'cookie': 'intellpositionL=1152px; em_hq_fls=js; intellpositionT=455px; qgqp_b_id=2c2759821488d533c53576ad44a41e5d; HAList=ty-1-000001-%u4E0A%u8BC1%u6307%u6570%2Cty-90-BK1036-%u534A%u5BFC%u4F53%2Ca-sh-603713-%u5BC6%u5C14%u514B%u536B%2Cf-0-000001-%u4E0A%u8BC1%u6307%u6570%2Ca-sh-600559-%u8001%u767D%u5E72%u9152%2Ca-sh-603195-%u516C%u725B%u96C6%u56E2%2Ca-sz-002236-%u5927%u534E%u80A1%u4EFD%2Ca-sh-600759-%u6D32%u9645%u6CB9%u6C14%2Cf-0-399439-%u56FD%u8BC1%u6CB9%u6C14%2Ca-sz-000049-%u5FB7%u8D5B%u7535%u6C60; EMFUND1=null; EMFUND2=null; EMFUND3=null; EMFUND4=null; EMFUND5=null; EMFUND6=null; EMFUND7=null; EMFUND0=null; EMFUND8=04-11%2015%3A07%3A30@%23%24%u534E%u5546%u4FE1%u7528%u589E%u5F3A%u503A%u5238A@%23%24001751; AUTH_FUND.EASTMONEY.COM_GSJZ=AUTH*TTJJ*TOKEN; EMFUND9=04-11 15:32:10@#$%u62DB%u5546%u4E2D%u8BC1%u767D%u9152%u6307%u6570%28LOF%29A@%23%24161725; st_si=97748096283622; st_pvi=48514956678552; st_sp=2019-09-26%2018%3A30%3A34; st_inirUrl=https%3A%2F%2Fwww.baidu.com%2Flink; st_sn=1; st_psi=2023041120385018-112200312942-2229925924; st_asi=delete',
    'host': 'api.fund.eastmoney.com',
    'referer': 'http://fund.eastmoney.com/',
    'useragent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36',
    'themecallback': 'jQuery18306592616272463223_1681284141591',
}

#基金信息
class FundInfo:
    #初始化
    def __init__(self, page_index = 1, page_size = 10, total_page = 0, total_count = 0):
        self.pageIndex = page_index
        self.pageSize = page_size
        self.totalPage = total_page
        self.totalCount = total_count
        self.dataList = []
        self.themeName = ''
    #请求接口
    def get_response(self, parame_obj, parame_url):
        # 伪装
        # Request Headers里的Cookie、Host、Referer、User-Agent。
        headers = {
            'Cookie': parame_obj['cookie'],
            'Host': parame_obj['host'],
            'Referer': parame_obj['referer'],
            'User-Agent': parame_obj['useragent'],
        }
        response = requests.get(url=parame_url, headers=headers)

        return response
    #获取基金主题
    def get_fund_theme(self, parame_obj):
        url = f'http://api.fund.eastmoney.com/ztjj//GetBKDetailInfoNew?callback={parame_obj["themecallback"]}&tp=801950&_=1681284141906'
        response = self.get_response(parame_obj, url)
        # 获取数据
        data = response.text
        print('theme===', data)
        # 解析数据
        res = re.findall(f'{parame_obj["themecallback"]}\((.*)\)', data)[0]
        res = json.loads(res)

        self.themeName = res['Data']['INDEXNAME']
    #获取基金列表数据
    def get_fund_data(self, parame_obj):
        # General里的Request URL
        # 将callback、sort、sorttype、pageindex、pagesize、isbuy这几个参数提取出来
        url = f'http://api.fund.eastmoney.com/ztjj/GetBKRelTopicFundNew?callback={parame_obj["callback"]}&sort={parame_obj["sort"]}&sorttype={parame_obj["sorttype"]}&pageindex={self.pageIndex}&pagesize={self.pageSize}&tp=801950&isbuy={parame_obj["isbuy"]}'
        response = self.get_response(parame_obj, url)
        # 获取数据
        data = response.text
        print(data)

        # 解析数据
        res = re.findall(f'{parame_obj["callback"]}\((.*)\)', data)[0]
        res = json.loads(res)
        return res
    #获取总条数、总页数
    def get_total_page(self, parame_res):
        self.totalCount = int(parame_res['TotalCount'])
        self.totalPage = math.ceil(self.totalCount / self.pageSize)
    def get_fund_info(self, parame_res):
        datas = parame_res['Data']
        for list in datas:
            itemList = []
            for key, val in list.items():
                if 'SHORTNAME' in key:
                    itemList.append(str(val))
                elif 'FCODE' in key:
                    itemList.append(str(val))
                elif 'DWJZ' in key:
                    itemList.append(str(val))
                elif 'RZDF' in key:
                    itemList.append(str(val))
                elif 'SYRQ' in key:
                    itemList.append(str(val))
                elif 'FTYPE' in key:
                    itemList.append(str(val))
                elif 'RELATION' in key:
                    itemList.append(str(val))
                elif 'SYL_Z' in key:
                    itemList.append(str(val))
                elif 'SYL_Y' in key:
                    itemList.append(str(val))
                elif 'SYL_3Y' in key:
                    itemList.append(str(val))
                elif 'SYL_6Y' in key:
                    itemList.append(str(val))
                elif 'SYL_JN' in key:
                    itemList.append(str(val))
                elif 'SYL_1N' in key:
                    itemList.append(str(val))
                elif 'SYL_2N' in key:
                    itemList.append(str(val))
                elif 'SYL_3N' in key:
                    itemList.append(str(val))
                elif 'SYL_LN' in key:
                    itemList.append(str(val))
                elif 'RATE' in key:
                    itemList.append(str(val))
                elif 'SOURCERATE' in key:
                    itemList.append(str(val))
                elif 'MINSG' in key:
                    itemList.append(str(val))
                elif 'ISSALES' in key:
                    itemList.append(str(val))
                elif 'ISBUY' in key:
                    itemList.append(str(val))
            self.dataList.append(itemList)
    #分页
    def change_page(self, parame_obj, parame_res):
        self.get_fund_theme(parame_obj)
        timestamp = int(time.time())
        for page in range(self.totalPage):
            print(f'正在爬取第{page+1}页的数据内容')
            time.sleep(1)
            self.get_fund_data(parame_obj)
            self.get_fund_info(parame_res)
            self.pageIndex += 1
            if self.pageIndex > self.totalPage:
                df = pd.DataFrame(self.dataList, columns=['基金代码', '', '日期', '原手续费', '净值', '日增长率', '手续费', 'ISBUY', '购买起点', '基金名称', '近1周','近1月','近3月','近6月','近1年','近2年','近3年','今年来','成立来', 'ISSALES','基金类型'])
                df.to_excel(f'{self.themeName}基金{timestamp}.xlsx')
                break

fund_info = FundInfo()
res = fund_info.get_fund_data(fund_obj)
fund_info.get_total_page(res)
fund_info.change_page(fund_obj, res)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140

运行上面的代码，然后就可以得到一份数据。