python3GUI--微博图片爬取工具V1.5（附源码）_微博爬虫gui程序

---

鉴于前面python3GUI–weibo图片爬取工具（附源码），简单实现了weibo大图爬取，简单的界面交互，本次在前篇上进行改进，精简代码量，增加用户搜索接口，展示用户头像，便于精准定位用户，准确爬取用户图片。

一．准备工作

基于Tkinter、requests
Tkinter为内置库，requests为第三方库，需要额外安装一下，具体不多介绍

二．预览

1.启动

去掉了相关图片、图标，界面更加精简。

2.搜索

通过调用weibo两个搜索接口，将搜索出来的用户展示在分组框，供用户选择，展示搜索出的第一个用户头像，便于用户确定用户。

3.运行中

4.结果

这里仅拿出一张图片用作结果展示

三．设计流程

1.总体设计

本程序主要操作流程

2.详细设计

Weibo_Crawl_Engine.py主要思路

四．源代码

1.Weibo_Pic_Crawl-GUI-V1.5.py

import io
import os
import threading
from PIL import  ImageTk
from PIL import Image as imim
from tkinter import ttk
from tkinter import *
from tkinter import messagebox
from tkinter.filedialog import *
from tkinter.scrolledtext import ScrolledText
from Weibo_Crawl_Engine import Weibo_Pic_Spider

class App:
    def __init__(self):
        self.root=Tk()
        self.root.title('weibo图片小爬虫-V1.5')
        self.root.resizable(0,0)
        width=380
        height=310
        left=(self.root.winfo_screenwidth()-width)/2
        top=(self.root.winfo_screenheight()-height)/2
        self.root.geometry('%dx%d+%d+%d'%(width,height,left,top))
        self.s=Weibo_Pic_Spider()
        self.create_widgets()
        self.set_widgets()
        self.place_widgets()
        self.root.mainloop()

    def create_widgets(self):
        self.label1=ttk.Label(self.root,text='关键字：')
        self.entry_keyword_var=StringVar()
        self.entry_keyword=ttk.Entry(self.root,textvariable=self.entry_keyword_var,justify='center')
        self.btn_search=ttk.Button(self.root,text='搜索',command=self.do_search_users)

        self.label_disk=ttk.Label(self.root,text='保存位置：')
        self.entry_savelocation_var=StringVar()
        self.entry_savelocation=ttk.Entry(self.root,textvariable=self.entry_savelocation_var,justify='center')
        self.check_opendir_var=BooleanVar(value=False)

        self.label_userlist=ttk.Label(self.root,text='用户列表：')
        self.combobox_userlist=ttk.Combobox(self.root,value=[],state='readonly',justify='center')
        self.btn_set_save_dir=ttk.Button(self.root,text='选择',command=self.select_save_dir)

        self.btn_startcrawl=ttk.Button(self.root,text='开始爬取',command=lambda :self.thread_it(self.do_download_pic))

        self.label_state=ttk.Label(self.root,text='状态：')
        self.scrolltext_state=ScrolledText(self.root)

        self.label_show_userimg=ttk.Label(self.root,text='用户头像',justify="center",compound='bottom',background='lightblue',width=20)

        self.lableframe=ttk.Labelframe(self.root,text='')
        self.lable_tip=ttk.Label(self.lableframe,text='敬告：本软件仅供学习交流使用！',foreground='red')

        self.m=Menu(self.root)
        self.root['menu'] = self.m
        self.s1=Menu(self.m,tearoff=False)
        self.s2=Menu(self.m,tearoff=False)
        self.s3=Menu(self.m,tearoff=False)

    def set_widgets(self):
        self.m.add_cascade(menu=self.s1,label='开始')
        self.m.add_cascade(menu=self.s2,label='操作')
        self.m.add_cascade(menu=self.s3,label='关于')
        self.s1.add_command(label='打开文件夹',command=lambda:os.startfile(self.abs_path))
        self.s1.add_separator()
        self.s1.add_command(label='退出',command=self.quit_window)

        self.combobox_userlist.bind("<<ComboboxSelected>>",self.show_user_head)
        self.entry_keyword.bind('<Return>',self.do_search_users)
        self.root.protocol("WM_DELETE_WINDOW",self.quit_window)
        self.open_dir_var=IntVar()
        #这里只能使用checkbutton用于开关
        self.s2.add_command(label='搜索',command=self.do_search_users)
        self.s2.add_command(label='开始爬取',command=lambda :self.thread_it(self.do_download_pic))
        self.s2.add_separator()
        self.s2.add_checkbutton(label='下载完后打开文件夹',variable=self.open_dir_var,command=self.open_dir_after_crawl)
        self.btn_startcrawl.config(state=DISABLED)
        self.s2.entryconfig("开始爬取", state=DISABLED)
        self.s3.add_command(label='关于作者',command=lambda :messagebox.showinfo('作者信息','作者：懷淰メ\nversion:1.5'))


    def place_widgets(self):
        self.label1.place(x=10,y=15)
        self.entry_keyword.place(x=80,y=15,width=190)
        self.btn_search.place(x=280,y=15)

        self.label_disk.place(x=10,y=55)
        self.entry_savelocation.place(x=80,y=55,width=190)
        self.btn_set_save_dir.place(x=280,y=55)

        self.label_userlist.place(x=10,y=95)
        self.combobox_userlist.place(x=80,y=95,width=190)
        self.btn_startcrawl.place(x=280,y=95)

        self.label_show_userimg.place(x=280,y=140,height=120)

        self.label_state.place(x=10,y=180)
        self.scrolltext_state.place(x=80,y=140,width=200,height=120)


        self.lableframe.place(x=80,y=260)
        self.lable_tip.pack(anchor="w",fill=X)

    def show_user_head(self,*args):
        current_index=self.combobox_userlist.current()
        paned_img=PanedWindow(self.label_show_userimg)
        head_img=self.search_result[current_index]['user_head_img'].split('?')[0]
        image_bytes=self.s.get_img_bytes(head_img)
        data_stream = io.BytesIO(image_bytes)
        pil_image = imim.open(data_stream)
        photo = pil_image.resize((95, 90))
        paned_img.image=ImageTk.PhotoImage(photo)
        self.label_show_userimg.config(image=paned_img.image,compound='bottom')

    def do_search_users(self,*args):
        self.combobox_userlist.config(value=[])
        try:
            self.search_result.clear()
        except AttributeError:
            pass
        key_word=self.entry_keyword_var.get().strip()
        if key_word !='':
            self.search_result=self.s.get_users(key_word)
            if self.search_result:
                user_names=[user_['user_name'] for user_ in self.search_result]
                self.combobox_userlist.config(value=user_names)
                self.btn_startcrawl.config(state=NORMAL)
                self.s2.entryconfig("开始爬取", state=NORMAL)
                self.combobox_userlist.current(0)
                self.show_user_head()
            else:
                messagebox.showinfo('提示',f'很抱歉，没有检索到关于[{key_word}]的用户！')
        else:
            messagebox.showwarning('警告','关键字不能为空！')

    def do_download_pic(self):
        try:
            if self.entry_savelocation_var.get()!='':
                current_item=self.search_result[self.combobox_userlist.current()]
                user_id=current_item['user_id']
                user_name=current_item['user_name']
                self.s.set_start_url(user_id)
                abs_path=self.entry_savelocation_var.get()+f'/{user_name}'
                self.abs_path=abs_path
                self.s.do_make_dirs(abs_path)
                for pic in self.s.get_pics_url():
                    file_name = pic.split('/')[-1]
                    self.s.do_download_pic(pic,file_name,abs_path)
                    self.insert_into_scrolltext(f"{file_name}下载完成...")
                else:
                    self.btn_startcrawl.config(state=NORMAL)
                    self.s2.entryconfig("开始爬取", state=NORMAL)
                    if self.open_dir_flag:
                        os.startfile(self.abs_path)
            else:
                messagebox.showwarning('警告', '请选择图片保存路径！')
                self.select_save_dir()
        except AttributeError :
            messagebox.showwarning('警告','请先搜索！')

    def select_save_dir(self):
        while True:
            save_dir=askdirectory(title = "选择一个图片保存目录", initialdir='./', mustexist = True)
            if save_dir:
                self.entry_savelocation_var.set(save_dir)
                break
            else:
                messagebox.showwarning('警告','请选择图片保存的文件夹！')


    def open_dir_after_crawl(self):
        #图片爬取结束后，打开文件夹
        if self.open_dir_var.get()==1:
            self.open_dir_flag=True
        else:
            pass

    def insert_into_scrolltext(self,line):
        self.scrolltext_state.insert(END,line+'\n')
        self.scrolltext_state.yview_moveto(1)

    def quit_window(self,*args):
        ret=messagebox.askyesno('退出','确定要退出？')
        if ret :
            self.root.destroy()
        else:
            pass

    def thread_it(self,func,*args):
        t=threading.Thread(target=func,args=args)
        t.setDaemon(True)
        t.start()

if __name__ == '__main__':
    a=App()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195

2.Weibo_Crawl_Engine.py

# coding:utf-8
import threading
import traceback

import requests
import json
import random
import os
from urllib.parse import quote
from lxml import etree
import urllib3

urllib3.disable_warnings()


class Weibo_Pic_Spider:
    def __init__(self):
        self.search_api1 = 'https://m.weibo.cn/api/container/getIndex?containerid=100103type%3D3%26q%3D{}%26t%3D0'  # 移动端API搜索
        self.search_api2 = 'https://s.weibo.com/user?q={}&Refer=weibo_user'  # 静态网页搜索
        self.header2 = {
            "Accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
            "Accept-Encoding": "gzip, deflate, br",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Referer": "https://weibo.com/",
            'Sec-Ch-Ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
            "Sec-Ch-Ua-Platform": "Windows",
            "Sec-Fetch-Dest": "image",
            "Sec-Fetch-Mode": "no-cors",
            "Sec-Fetch-Site": "cross-site",
            "User-Agent": self.get_ua(),
        }
        self.user_list = []

    def get_user_list1(self, key_word):
        full_url1 = self.search_api1.format(key_word)
        header2 = {'User-Agent': self.get_ua()}
        r = requests.get(full_url1, headers=header2)
        if r.status_code == 200:
            try:
                _json = json.loads(r.text)
                users = _json['data']['cards'][1].get('card_group')
                for user_ in users:
                    item = {}
                    user_info = user_.get('user')
                    item['user_name'] = user_info.get('screen_name')
                    item['user_id'] = user_info.get('id')
                    item['user_head_img'] = user_info.get('profile_image_url')
                    self.user_list.append(item)
            except:
                traceback.print_exc()

    # 解析出图片地址
    def get_user_list2(self, key_word):
        current_username = [user["user_name"] for user in self.user_list]
        full_url2 = self.search_api2.format(quote(key_word))
        r = requests.get(full_url2, headers=self.header2)
        if r.status_code == 200:
            try:
                res = etree.HTML(r.text)
                selector = res.xpath('//div[@id="pl_user_feedList"]/div')
                for data in selector:
                    item = {}
                    user_name = ''.join(data.xpath('.//div/a[1]//text()'))
                    if user_name in current_username:
                        # 根据API1获取的用户在这里不给予显示
                        continue
                    item['user_name'] = user_name
                    item['user_id'] = ''.join(data.xpath('./div[@class="info"]/div/a[@uid]/@uid'))
                    item['user_head_img'] = ''.join(data.xpath('.//div[@class="avator"]/a/img/@src'))
                    self.user_list.append(item)
            except:
                traceback.print_exc()

    def get_users(self, key_word):
        """
        通过API1和API2获取检索的用户信息
        :param key_word:
        :return:
        """
        self.get_user_list1(key_word)
        self.get_user_list2(key_word)
        if len(self.user_list) != 0:
            return self.user_list
        else:
            return False

    def set_start_url(self, user_id):
        container_id = '107603' + str(user_id)
        start_url = f'https://m.weibo.cn/api/container/getIndex?containerid={container_id}'
        self.start_url = start_url

    def get_pics_url(self):
        i = 1
        while True:
            url = self.start_url + '&page={}'.format(i)
            header2 = {'User-Agent': self.get_ua()}
            r = requests.get(url, headers=header2)
            _json = json.loads(r.text)
            items = _json["data"]["cards"]
            flag = _json['ok']
            if flag == 1:
                for v in items:
                    picslist = v.get('mblog')
                    if picslist is not None:
                        img_urls = picslist.get('pics')
                        if img_urls != None:
                            for img_url_ in img_urls:
                                img_url = img_url_['large']['url']
                                yield img_url
            else:
                break
            i += 1

    def get_ua(self):
        first_num = random.randint(55, 62)
        third_num = random.randint(0, 3200)
        fourth_num = random.randint(0, 140)
        os_type = [
            '(Windows NT 6.1; WOW64)', '(Windows NT 10.0; WOW64)', '(X11; Linux x86_64)',
            '(Macintosh; Intel Mac OS X 10_12_6)'
        ]
        chrome_version = 'Chrome/{}.0.{}.{}'.format(first_num, third_num, fourth_num)

        ua = ' '.join(['Mozilla/5.0', random.choice(os_type), 'AppleWebKit/537.36',
                       '(KHTML, like Gecko)', chrome_version, 'Safari/537.36']
                      )
        return ua

    def do_make_dirs(self, dir_path):
        try:
            os.makedirs(dir_path)
        except:
            pass

    def do_download_pic(self, pic_url, file_name, base_dir):
        r = requests.get(pic_url, headers=self.header2, verify=False)
        with open(base_dir + '/' + file_name, 'wb')as f:
            f.write(r.content)

    def get_img_bytes(self, img_url):
        try:
            r = requests.get(img_url, headers=self.header2)
            if r.status_code == 200:
                pic_bytes = r.content
                return pic_bytes
            else:
                return False
        except requests.exceptions:
            return False


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151

五．总结说明

本次在原版（V1.0版本）进行升级改进，仍旧实现weibo图片高清原图的解析下载，但是相比之前版本主要提升在于：

1.去除无用功能，精简代码
2.将GUI于爬虫分离开来，解耦
3.在用户检索上，加入了网页版用户搜索API，增加了可被检索的用户数量
4.选择要用户后，会显示指定头像
5.优化程序运行输出逻辑

程序打包好放在了蓝奏云，思路、代码方面有什么不足欢迎各位大佬指正、批评！

PS：2023年6月17日 更新了Weibo_Crawl_Engine.py