1 # -*- coding: utf-8 -*- 2 """ 3 Created on Thu Oct 18 09:13:32 2018 4 5 @author: Gawen 6 7 实现爬取IEEE目标网页上该页所有论文的摘要 8 并通过百度翻译api进行翻译 9 并将链接与翻译后的摘要存到文本文档中 10 其中百度的api的appid以及secertkey需要自己申请 11 每月有200万字符的免费额度 12 python版本3.6 13 需要的包:selenium 14 需要的软件:chrome-driver 15 """ 16 #example url https://ieeexplore.ieee.org/xpl/mostRecentIssue.jsp?punumber=8360187&punumber=8360187&filter=issueId%20EQ%20%228363090%22&pageNumber=9&pageNumber=10 17 import requests 18 from bs4 import BeautifulSoup 19 from selenium import webdriver 20 import time 21 from selenium.webdriver.chrome.options import Options 22 import hashlib 23 import urllib 24 import random 25 import json 26 27 28 def writetxt(file,url,abstract): 29 with open(file, 'a', encoding='GBK') as file_txt: 30 file_txt.write('链接:\n'+url) 31 file_txt.write('\n') 32 file_txt.write('摘要：\n'+abstract) 33 file_txt.write('\n') 34 file_txt.write('\n') 35 36 37 def trans(q): 38 appid = ''#你自己的百度翻译appid 39 secretkey = ''#你自己的百度翻译secretkey 40 myurl = '/api/trans/vip/translate' 41 fromLang = 'en' 42 toLang = 'zh' 43 salt = random.randint(32768,65536) 44 sign = appid + q + str(salt) + secretkey 45 sign = hashlib.md5(sign.encode(encoding = 'utf-8')).hexdigest() 46 myurl = myurl + '?appid=' + appid + '&q=' + urllib.parse.quote(q) + '&from=' + fromLang + '&to=' + toLang + '&salt=' + str(salt) + '&sign=' + sign 47 print(myurl) 48 try: 49 r = requests.get('http://api.fanyi.baidu.com'+myurl) 50 print(r.content.decode('utf-8')) 51 except Exception as e: 52 print(e) 53 html = r.content.decode('utf-8') 54 soup = BeautifulSoup(html,'lxml') 55 text = soup.find('p').text 56 text_dict = json.loads(text) 57 if ('error_code' in text_dict.keys()): 58 return 'error' 59 return text_dict['trans_result'][0]['dst'] 60 61 62 url = input('please input the url that you want to download:\n') 63 fore = 'https://ieeexplore.ieee.org' 64 r = requests.get(url) 65 html = r.content.decode('utf-8') 66 soup = BeautifulSoup(html,'lxml') 67 h3 = soup.find('div', class_='cf jrnl-results-filter').find_all('h3') 68 h3text = [] 69 errtitle = [] 70 links = [] 71 for h in h3: 72 h3text.append(h.text.strip()) 73 print(h3text) 74 for i in range(len(h3text)): 75 if ((soup.find('a', attrs={'aria-label':'View HTML: ' + h3text[i]}))==None): 76 errtitle.append(h3text[i]) 77 continue 78 href = (soup.find('a', attrs={'aria-label':'View HTML: ' + h3text[i]})['href']) 79 links.append(fore + href) 80 print(links) 81 chrome_options = Options() 82 chrome_options.add_argument('--headless') 83 chrome_options.add_argument('--disable-gpu') 84 driver = webdriver.Chrome(chrome_options=chrome_options) 85 count = 0 86 for link in links: 87 driver.get(link) 88 driver.implicitly_wait(20) 89 ps = driver.page_source 90 lsoup = BeautifulSoup(ps,'lxml') 91 abstract = lsoup.select('body > div > div > div > div > div > div > xpl-root > xpl-document-details > div > div > div > div > section > div > div > xpl-document-abstract > section > div > div > div > div > div')[0].text 92 abstract = trans(abstract) 93 if(abstract=='error'): 94 errtitle.append(link) 95 continue 96 writetxt(r'C:\Users\Gawen\Desktop\abstract.txt',link,abstract)#输出路径 97 count += 1 98 print(count) 99 time.sleep(5) 100 driver.close() 101 print("共有"+str(len(errtitle))+"篇论文下载失败") 102 for err in errtitle: 103 print(err) 104 105 106
