devbox
盐析白兔
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

爬取百度相关搜索结果_百度搜索结果提取

作者：盐析白兔 | 2024-06-29 04:19:50

赞

踩

百度搜索结果提取

背景：

有一大堆query，需要获取在百度上的搜索结果。然后整理到Excel上。

代码：

项目结构如下：

一共有两个脚本文件 paqushuju.py 和 threadingdata.py

paqushuju.py 是单线程脚本

 threadingdata.py 是多线程脚本

paqushuju.py 如下：

  1. # -*- coding:utf-8 -*-
  2. # paqushuju.py 
  3. from selenium import webdriver
  4. import time
  5. import xlrd
  6. import xlsxwriter
  7. import time
  8. import os
  9.  
  10. # 读取 query2文件夹下所有的文件，list
  11. query_path = r'D:\babytree\codetest\zuoye\pachong\query2'
  12. wenjianlist = os.listdir(query_path)
  13.  
  14. # 给生成的新的文件，命名
  15. namelist = []
  16. PathList = []
  17. for single in wenjianlist:
  18.     newsingle = single.split('.')[0] + '的搜索结果.xlsx'
  19.     namelist.append(newsingle)
  20.     # 要读取的每个文件的位置
  21.     PathList.append(r'D:\babytree\codetest\zuoye\pachong\query2' + "\\" + single)
  22.  
  23. for namei in range(len(wenjianlist)):
  24.     time.sleep(5)
  25.  
  26.     ###############################################
  27.     # 初始化浏览器
  28.     browser = webdriver.Chrome(r'D:\python3\Lib\chromedriver.exe')
  29.     browser.get('https://www.baidu.com/')
  30.     browser.maximize_window()
  31.     browser.find_element_by_xpath('//*[@id="kw"]').send_keys(u'怎么样看早孕试纸11')
  32.     browser.find_element_by_xpath('//*[@id="su"]').click()
  33.     time.sleep(1)
  34.     browser.find_element_by_id('kw').clear()
  35.  
  36.     ##################################################
  37.     # 创建一个workbook 设置编码
  38.     workbook = xlsxwriter.Workbook(namelist[namei])
  39.  
  40.     #####################################################
  41.     # 数据初始化
  42.  
  43.     # 文件地址+名称
  44.     thePath = PathList[namei]
  45.  
  46.     # 读取第几列，这是一个列表
  47.     dataColumn = []
  48.     # 读取名称是第几列的数据
  49.     dataCouName = '点击位置'
  50.  
  51.     # 打开文件
  52.     data = xlrd.open_workbook(thePath)
  53.  
  54.     # 所有的sheet页，是一个list
  55.     sheetlist = data.sheet_names()
  56.     for sheetNamei in range(len(sheetlist)):
  57.         # 创建一个worksheet
  58.         worksheet = workbook.add_worksheet(sheetlist[sheetNamei])
  59.  
  60.         # 循环读取的sheet页
  61.  
  62.         # 通过文件名获得工作表,获取工作表
  63.         table = data.sheet_by_name(sheetlist[sheetNamei])
  64.         # 获取第一列数据
  65.         # 第一列数据，是个list，然后干掉列表第一个
  66.         fistCol = table.col_values(0)
  67.         if fistCol[0] == 'QUERY' or fistCol[0] == 'yunyu':
  68.             fistCol.pop(0)
  69.         else:
  70.             pass
  71.  
  72.         for searchKyi in range(len(fistCol)):
  73.             browser.implicitly_wait(15)
  74.             try:
  75.                 browser.find_element_by_id('kw').send_keys(fistCol[searchKyi])
  76.  
  77.             except:
  78.                 time.sleep(2)
  79.                 browser.find_element_by_id('kw').send_keys(fistCol[searchKyi])
  80.  
  81.             browser.implicitly_wait(15)
  82.             try:
  83.                 browser.find_element_by_id('su').click()
  84.             except:
  85.                 time.sleep(2)
  86.                 browser.find_element_by_id('su').click()
  87.             browser.implicitly_wait(15)
  88.             browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
  89.             time.sleep(6)
  90.  
  91.             try:
  92.                 a = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[1]/th[1]/a').text
  93.             except:
  94.                 a = '无相关数据'
  95.             try:
  96.                 b = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[1]/th[2]/a').text
  97.             except:
  98.                 b = '无相关数据'
  99.             try:
  100.                 c = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[1]/th[3]/a').text
  101.             except:
  102.                 c = '无相关数据'
  103.             try:
  104.                 d = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[2]/th[1]/a').text
  105.             except:
  106.                 d = '无相关数据'
  107.             try:
  108.  
  109.                 e = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[2]/th[2]/a').text
  110.             except:
  111.                 e = '无相关数据'
  112.  
  113.             try:
  114.                 f = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[2]/th[3]/a').text
  115.             except:
  116.                 f = '无相关数据'
  117.             try:
  118.                 g = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[3]/th[1]/a').text
  119.  
  120.             except:
  121.                 g = '无相关数据'
  122.             try:
  123.                 h = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[3]/th[2]/a').text
  124.             except:
  125.                 h = '无相关数据'
  126.             try:
  127.                 i = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[3]/th[3]/a').text
  128.             except:
  129.                 i = '无相关数据'
  130.  
  131.             ai = fistCol[searchKyi]
  132.             searchData = [ai, a, b, c, d, e, f, g, h, i]
  133.             for m in range(len(searchData)):
  134.                 worksheet.write(searchKyi, m, searchData[m])
  135.                 print(searchData[m])
  136.             browser.implicitly_wait(15)
  137.             browser.find_element_by_id('kw').clear()
  138.  
  139.     # 保存
  140.     workbook.close()

threadingdata.py 代码如下

  1. # -*- coding:utf-8 -*-
  2. # threadingdata.py
  3. from selenium import webdriver
  4. import time
  5. import xlrd
  6. import xlsxwriter
  7. import time
  8. import os
  9. import threading
  10.  
  11.  
  12. def crawler(query_path):
  13.     wenjianlist = os.listdir(query_path)
  14.  
  15.     # 给生成的新的文件，命名
  16.     namelist = []
  17.     PathList = []
  18.     for single in wenjianlist:
  19.         newsingle = single.split('.')[0] + '的搜索结果.xlsx'
  20.         namelist.append(newsingle)
  21.         # 要读取的每个文件的位置
  22.         PathList.append(query_path + "\\" + single)
  23.  
  24.     for namei in range(len(wenjianlist)):
  25.         time.sleep(5)
  26.  
  27.         ###############################################
  28.         # 初始化浏览器
  29.         browser = webdriver.Chrome(r'D:\python3\Lib\chromedriver.exe')
  30.         browser.get('https://www.baidu.com/')
  31.         browser.maximize_window()
  32.         browser.find_element_by_xpath('//*[@id="kw"]').send_keys(u'怎么样看早孕试纸11')
  33.         browser.find_element_by_xpath('//*[@id="su"]').click()
  34.         time.sleep(1)
  35.         browser.find_element_by_id('kw').clear()
  36.  
  37.         ##################################################
  38.         # 创建一个workbook 设置编码
  39.         workbook = xlsxwriter.Workbook(namelist[namei])
  40.  
  41.         #####################################################
  42.         # 数据初始化
  43.  
  44.         # 文件地址+名称
  45.         thePath = PathList[namei]
  46.  
  47.         # 读取第几列，这是一个列表
  48.         dataColumn = []
  49.         # 读取名称是第几列的数据
  50.         dataCouName = '点击位置'
  51.  
  52.         # 打开文件
  53.         data = xlrd.open_workbook(thePath)
  54.  
  55.         # 所有的sheet页，是一个list
  56.         sheetlist = data.sheet_names()
  57.         for sheetNamei in range(len(sheetlist)):
  58.             # 创建一个worksheet
  59.             worksheet = workbook.add_worksheet(sheetlist[sheetNamei])
  60.  
  61.             # 循环读取的sheet页
  62.  
  63.             # 通过文件名获得工作表,获取工作表
  64.             table = data.sheet_by_name(sheetlist[sheetNamei])
  65.             # 获取第一列数据
  66.             # 第一列数据，是个list，然后干掉列表第一个
  67.             fistCol = table.col_values(0)
  68.             if fistCol[0] == 'QUERY' or fistCol[0] == 'yunyu':
  69.                 fistCol.pop(0)
  70.             else:
  71.                 pass
  72.  
  73.             for searchKyi in range(len(fistCol)):
  74.                 browser.implicitly_wait(15)
  75.                 try:
  76.                     browser.find_element_by_id('kw').send_keys(fistCol[searchKyi])
  77.  
  78.                 except:
  79.                     time.sleep(2)
  80.                     browser.find_element_by_id('kw').send_keys(fistCol[searchKyi])
  81.  
  82.                 browser.implicitly_wait(15)
  83.                 try:
  84.                     browser.find_element_by_id('su').click()
  85.                 except:
  86.                     time.sleep(2)
  87.                     browser.find_element_by_id('su').click()
  88.                 browser.implicitly_wait(15)
  89.                 browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
  90.                 time.sleep(6)
  91.  
  92.                 try:
  93.                     a = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[1]/th[1]/a').text
  94.                 except:
  95.                     a = '无相关数据'
  96.                 try:
  97.                     b = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[1]/th[2]/a').text
  98.                 except:
  99.                     b = '无相关数据'
  100.                 try:
  101.                     c = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[1]/th[3]/a').text
  102.                 except:
  103.                     c = '无相关数据'
  104.                 try:
  105.                     d = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[2]/th[1]/a').text
  106.                 except:
  107.                     d = '无相关数据'
  108.                 try:
  109.  
  110.                     e = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[2]/th[2]/a').text
  111.                 except:
  112.                     e = '无相关数据'
  113.  
  114.                 try:
  115.                     f = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[2]/th[3]/a').text
  116.                 except:
  117.                     f = '无相关数据'
  118.                 try:
  119.                     g = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[3]/th[1]/a').text
  120.  
  121.                 except:
  122.                     g = '无相关数据'
  123.                 try:
  124.                     h = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[3]/th[2]/a').text
  125.                 except:
  126.                     h = '无相关数据'
  127.                 try:
  128.                     i = browser.find_element_by_xpath('//*[@id="rs"]/div/table/tbody/tr[3]/th[3]/a').text
  129.                 except:
  130.                     i = '无相关数据'
  131.  
  132.                 ai = fistCol[searchKyi]
  133.                 searchData = [ai, a, b, c, d, e, f, g, h, i]
  134.                 for m in range(len(searchData)):
  135.                     worksheet.write(searchKyi, m, searchData[m])
  136.                     print(searchData[m])
  137.                 browser.implicitly_wait(15)
  138.                 browser.find_element_by_id('kw').clear()
  139.  
  140.         # 保存
  141.         workbook.close()
  142.         browser.quit()
  143.  
  144.  
  145. # 读取 query2文件夹下所有的文件，list
  146. query_path1 = r'D:\babytree\codetest\zuoye\pachong\query2'
  147. query_path2 = r'D:\babytree\codetest\zuoye\pachong\query3'
  148. query_path3 = r'D:\babytree\codetest\zuoye\pachong\query4'
  149. query_path4 = r'D:\babytree\codetest\zuoye\pachong\query5'
  150. # query_path5 = r'D:\babytree\codetest\zuoye\pachong\query6'
  151. # query_path6 = r'D:\babytree\codetest\zuoye\pachong\query7'
  152. threads = []
  153. t1 = threading.Thread(target=crawler, args=(query_path1,))
  154. threads.append(t1)
  155. t2 = threading.Thread(target=crawler, args=(query_path2,))
  156. threads.append(t2)
  157. t3 = threading.Thread(target=crawler, args=(query_path3,))
  158. threads.append(t3)
  159. t4 = threading.Thread(target=crawler, args=(query_path4,))
  160. threads.append(t4)
  161. # t5 = threading.Thread(target=crawler,args=(query_path5,))
  162. # threads.append(t5)
  163. # t6 = threading.Thread(target=crawler,args=(query_path6,))
  164. # threads.append(t6)
  165. if __name__ == '__main__':
  166.     for t in threads:
  167.         t.setDaemon(True)
  168.         t.start()
  169.  
  170.     for t in threads:
  171.         t.join()

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/盐析白兔/article/detail/768233
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号