devbox
很楠不爱3
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

Python爬取京东商品信息以及评论存进MySQL_怎么用python爬去京东的商品信息和价格并存储到mysql

作者：很楠不爱3 | 2024-04-04 15:21:59

赞

踩

怎么用python爬去京东的商品信息和价格并存储到mysql

目录

构建mysql数据表

第一版：

第二版 ：

第三版：

总结：

构建mysql数据表

问题：使用SQL alchemy时，非主键不能设置为自增长，但是我想让这个非主键仅仅是为了作为索引，autoincrement=True无效，该怎么实现让它自增长呢？

  1. from sqlalchemy import String,Integer,Text,Column
  2. from sqlalchemy import create_engine
  3. from sqlalchemy.orm import sessionmaker
  4. from sqlalchemy.orm import scoped_session
  5. from sqlalchemy.ext.declarative import declarative_base
  6.  
  7. engine=create_engine(
  8.     "mysql+pymysql://root:root@127.0.0.1:3306/jdcrawl?charset=utf8",
  9.     pool_size=200,
  10.     max_overflow=300,
  11.     echo=False
  12. )
  13.  
  14. BASE=declarative_base() # 实例化
  15.  
  16. class Goods(BASE):
  17.     __tablename__='goods'
  18.     id=Column(Integer(),primary_key=True,autoincrement=True)
  19.     sku_id = Column(String(200), primary_key=True, autoincrement=False)
  20.     name=Column(String(200))
  21.     price=Column(String(200))
  22.     comments_num=Column(Integer)
  23.     shop=Column(String(200))
  24.     link=Column(String(200))
  25.  
  26. class Comments(BASE):
  27.     __tablename__='comments'
  28.     id=Column(Integer(),primary_key=True,autoincrement=True,nullable=False)
  29.     sku_id=Column(String(200),primary_key=True,autoincrement=False)
  30.     comments=Column(Text())
  31.  
  32. BASE.metadata.create_all(engine)
  33. Session=sessionmaker(engine)
  34. sess_db=scoped_session(Session)

第一版：

问题：爬取几页评论后就会爬取到空白页，添加refer后依旧如此

尝试解决方法：将获取评论地方的线程池改为单线程，并每获取一页评论增加延时1s

  1. # 不能爬太快！！！不然获取不到评论
  2.  
  3. from bs4 import BeautifulSoup
  4. import requests
  5. from urllib import parse
  6. import csv,json,re
  7. import threadpool
  8. import time
  9. from jd_mysqldb import Goods,Comments,sess_db
  10.  
  11. headers={
  12.     'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
  13.     'Cookie': '__jdv=76161171|baidu|-|organic|%25E4%25BA%25AC%25E4%25B8%259C|1613711947911; __jdu=16137119479101182770449; areaId=7; ipLoc-djd=7-458-466-0; PCSYCityID=CN_410000_0_0; shshshfpa=07383463-032f-3f99-9d40-639cb57c6e28-1613711950; shshshfpb=u8S9UvxK66gfIbM1mUNrIOg%3D%3D; user-key=153f6b4d-0704-4e56-82b6-8646f3f0dad4; cn=0; shshshfp=9a88944b34cb0ff3631a0a95907b75eb; __jdc=122270672; 3AB9D23F7A4B3C9B=SEELVNXBPU7OAA3UX5JTKR5LQADM5YFJRKY23Z6HDBU4OT2NWYGX525CKFFVHTRDJ7Q5DJRMRZQIQJOW5GVBY43XVI; jwotest_product=99; __jda=122270672.16137119479101182770449.1613711948.1613738165.1613748918.4; JSESSIONID=C06EC8D2E9384D2628AE22B1A6F9F8FC.s1; shshshsID=ab2ca3143928b1b01f6c5b71a15fcebe_5_1613750374847; __jdb=122270672.5.16137119479101182770449|4.1613748918',
  14.     'Referer': 'https://www.jd.com/'
  15. }
  16.  
  17. num=0   # 商品数量
  18. comments_num=0   # 评论数量
  19.  
  20. # 获取商品信息和SkuId
  21. def getIndex(url):
  22.     session=requests.Session()
  23.     session.headers=headers
  24.     global num
  25.     res=session.get(url,headers=headers)
  26.     print(res.status_code)
  27.     res.encoding=res.apparent_encoding
  28.     soup=BeautifulSoup(res.text,'lxml')
  29.     items=soup.select('li.gl-item')
  30.     for item in items[:3]:  # 爬取3个商品测试
  31.         title=item.select_one('.p-name a em').text.strip().replace(' ','')
  32.         price=item.select_one('.p-price strong').text.strip().replace('￥','')
  33.         try:
  34.             shop=item.select_one('.p-shopnum a').text.strip()   # 获取书籍时查找店铺的方法
  35.         except:
  36.             shop=item.select_one('.p-shop a').text.strip()  #   获取其他商品时查找店铺的方法
  37.         link=parse.urljoin('https://',item.select_one('.p-img a').get('href'))
  38.         SkuId=re.search('\d+',link).group()
  39.         comments_num=getCommentsNum(SkuId,session)
  40.         print(SkuId,title, price, shop, link, comments_num)
  41.         print("开始存入数据库...")
  42.         try:
  43.             IntoGoods(SkuId,title, price, shop, link, comments_num)
  44.         except Exception as e:
  45.             print(e)
  46.             sess_db.rollback()
  47.         num += 1
  48.         print("正在获取评论...")
  49.         # 获取评论总页数
  50.         url1 = f'https://club.jd.com/comment/productPageComments.action?productId={SkuId}&score=0&sortType=5&page=0&pageSize=10'
  51.         headers['Referer'] = f'https://item.jd.com/{SkuId}.html'
  52.         headers['Connection']='keep-alive'
  53.         res2 = session.get(url1,headers=headers)
  54.         res2.encoding = res2.apparent_encoding
  55.         json_data = json.loads(res2.text)
  56.         max_page = json_data['maxPage']  # 经测试最多可获取100页评论，每页10条
  57.         args = []
  58.         for i in range(0, max_page):
  59.             # 使用此链接获取评论得到的为json格式
  60.             url2 = f'https://club.jd.com/comment/productPageComments.action?productId={SkuId}&score=0&sortType=5&page={i}&pageSize=10'
  61.             # 使用此链接获取评论得到的非json格式，需要提取
  62.             # url2_2=f'https://club.jd.com/comment/productPageComments.action?callback=jQuery9287224&productId={SkuId}&score=0&sortType=5&page={i}&pageSize=10'
  63.             args.append(([session,SkuId,url2], None))
  64.         pool2 = threadpool.ThreadPool(2)   # 2个线程
  65.         reque2 = threadpool.makeRequests(getComments,args)  # 创建任务
  66.         for r in reque2:
  67.             pool2.putRequest(r) # 提交任务到线程池
  68.         pool2.wait()
  69.  
  70. # 获取评论总数量
  71. def getCommentsNum(SkuId,sess):
  72.     headers['Referer']=f'https://item.jd.com/{SkuId}.html'
  73.     url=f'https://club.jd.com/comment/productCommentSummaries.action?referenceIds={SkuId}'
  74.     res=sess.get(url,headers=headers)
  75.     try:
  76.         res.encoding=res.apparent_encoding
  77.         json_data=json.loads(res.text)  # json格式转为字典
  78.         num=json_data['CommentsCount'][0]['CommentCount']
  79.         return num
  80.     except:
  81.         return 'Error'
  82.  
  83. # 获取评论
  84. def getComments(sess,SkuId,url2):
  85.     global comments_num
  86.     print(url2)
  87.     headers['Referer'] = f'https://item.jd.com/{SkuId}.html'
  88.     res2 = sess.get(url2,headers=headers)
  89.     res2.encoding='gbk'
  90.     json_data=res2.text
  91.     '''
  92.     # 如果用url2_2需要进行如下操作提取json
  93.     start = res2.text.find('jQuery9287224(') + len('jQuery9287224(')
  94.     end = res2.text.find(');')
  95.     json_data=res2.text[start:end]
  96.     '''
  97.     dict_data = json.loads(json_data)
  98.     try:
  99.         comments=dict_data['comments']
  100.         for item in comments:
  101.             comment=item['content'].replace('\n','')
  102.             # print(comment)
  103.             comments_num+=1
  104.             try:
  105.                 IntoComments(SkuId,comment)
  106.             except Exception as e:
  107.                 print(e)
  108.                 sess_db.rollback()
  109.     except:
  110.         pass
  111.  
  112. # 商品信息入库
  113. def IntoGoods(SkuId,title, price, shop, link, comments_num):
  114.     goods_data=Goods(
  115.         sku_id=SkuId,
  116.         name=title,
  117.         price=price,
  118.         comments_num=comments_num,
  119.         shop=shop,
  120.         link=link
  121.     )
  122.     sess_db.add(goods_data)
  123.     sess_db.commit()
  124.  
  125. # 评论入库
  126. def IntoComments(SkuId,comment):
  127.     comments_data=Comments(
  128.         sku_id=SkuId,
  129.         comments=comment
  130.     )
  131.     sess_db.add(comments_data)
  132.     sess_db.commit()
  133.  
  134. if __name__ == '__main__':
  135.     start_time=time.time()
  136.     urls=[]
  137.     KEYWORD=parse.quote(input("请输入要查询的关键词："))
  138.     for i in range(1,2):    # 爬取一页进行测试
  139.         url=f'https://search.jd.com/Search?keyword={KEYWORD}&wq={KEYWORD}&page={i}'
  140.         urls.append(([url,],None))  # threadpool要求必须这样写
  141.     pool=threadpool.ThreadPool(2)  # 2个线程的线程池
  142.     reque=threadpool.makeRequests(getIndex,urls)    # 创建任务
  143.     for r in reque:
  144.         pool.putRequest(r)  # 向线程池提交任务
  145.     pool.wait() # 等待所有任务执行完毕
  146.     print("共获取{}件商品，获得{}条评论，耗时{}".format(num,comments_num,time.time()-start_time))

第二版 ：

经测试，的确不会出现空白页的情况

进一步优化：同时获取2个以上商品的评论

  1. # 不能爬太快！！！不然获取不到评论
  2. from bs4 import BeautifulSoup
  3. import requests
  4. from urllib import parse
  5. import csv,json,re
  6. import threadpool
  7. import time
  8. from jd_mysqldb import Goods,Comments,sess_db
  9.  
  10. headers={
  11.     'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
  12.     'Cookie': '__jdv=76161171|baidu|-|organic|%25E4%25BA%25AC%25E4%25B8%259C|1613711947911; __jdu=16137119479101182770449; areaId=7; ipLoc-djd=7-458-466-0; PCSYCityID=CN_410000_0_0; shshshfpa=07383463-032f-3f99-9d40-639cb57c6e28-1613711950; shshshfpb=u8S9UvxK66gfIbM1mUNrIOg%3D%3D; user-key=153f6b4d-0704-4e56-82b6-8646f3f0dad4; cn=0; shshshfp=9a88944b34cb0ff3631a0a95907b75eb; __jdc=122270672; 3AB9D23F7A4B3C9B=SEELVNXBPU7OAA3UX5JTKR5LQADM5YFJRKY23Z6HDBU4OT2NWYGX525CKFFVHTRDJ7Q5DJRMRZQIQJOW5GVBY43XVI; jwotest_product=99; __jda=122270672.16137119479101182770449.1613711948.1613738165.1613748918.4; JSESSIONID=C06EC8D2E9384D2628AE22B1A6F9F8FC.s1; shshshsID=ab2ca3143928b1b01f6c5b71a15fcebe_5_1613750374847; __jdb=122270672.5.16137119479101182770449|4.1613748918',
  13.     'Referer': 'https://www.jd.com/'
  14. }
  15.  
  16. num=0   # 商品数量
  17. comments_num=0   # 评论数量
  18.  
  19. # 获取商品信息和SkuId
  20. def getIndex(url):
  21.     session=requests.Session()
  22.     session.headers=headers
  23.     global num
  24.     res=session.get(url,headers=headers)
  25.     print(res.status_code)
  26.     res.encoding=res.apparent_encoding
  27.     soup=BeautifulSoup(res.text,'lxml')
  28.     items=soup.select('li.gl-item')
  29.     for item in items[:2]:  # 爬取2个商品测试
  30.         title=item.select_one('.p-name a em').text.strip().replace(' ','')
  31.         price=item.select_one('.p-price strong').text.strip().replace('￥','')
  32.         try:
  33.             shop=item.select_one('.p-shopnum a').text.strip()   # 获取书籍时查找店铺的方法
  34.         except:
  35.             shop=item.select_one('.p-shop a').text.strip()  #   获取其他商品时查找店铺的方法
  36.         link=parse.urljoin('https://',item.select_one('.p-img a').get('href'))
  37.         SkuId=re.search('\d+',link).group()
  38.         headers['Referer'] = f'https://item.jd.com/{SkuId}.html'
  39.         headers['Connection'] = 'keep-alive'
  40.         comments_num=getCommentsNum(SkuId,session)
  41.         print(SkuId,title, price, shop, link, comments_num)
  42.         print("开始将商品存入数据库...")
  43.         try:
  44.             IntoGoods(SkuId,title, price, shop, link, comments_num)
  45.         except Exception as e:
  46.             print(e)
  47.             sess_db.rollback()
  48.         num += 1
  49.         print("正在获取评论...")
  50.         # 获取评论总页数
  51.         url1 = f'https://club.jd.com/comment/productPageComments.action?productId={SkuId}&score=0&sortType=5&page=0&pageSize=10'
  52.         res2 = session.get(url1,headers=headers)
  53.         res2.encoding = res2.apparent_encoding
  54.         json_data = json.loads(res2.text)
  55.         max_page = json_data['maxPage']  # 经测试最多可获取100页评论，每页10条
  56.         print("{}评论共{}页".format(SkuId,max_page))
  57.         if max_page==0:
  58.             IntoComments(SkuId,'0')
  59.         else:
  60.             for i in range(0, max_page):
  61.                 # 使用此链接获取评论得到的为json格式
  62.                 url2 = f'https://club.jd.com/comment/productPageComments.action?productId={SkuId}&score=0&sortType=5&page={i}&pageSize=10'
  63.                 # 使用此链接获取评论得到的非json格式，需要提取
  64.                 # url2_2=f'https://club.jd.com/comment/productPageComments.action?callback=jQuery9287224&productId={SkuId}&score=0&sortType=5&page={i}&pageSize=10'
  65.                 print("开始获取第{}页评论:{}".format(i+1,url2) )
  66.                 getComments(session,SkuId,url2)
  67.                 time.sleep(1)
  68.  
  69. # 获取评论总数量
  70. def getCommentsNum(SkuId,sess):
  71.     url=f'https://club.jd.com/comment/productCommentSummaries.action?referenceIds={SkuId}'
  72.     res=sess.get(url)
  73.     try:
  74.         res.encoding=res.apparent_encoding
  75.         json_data=json.loads(res.text)  # json格式转为字典
  76.         num=json_data['CommentsCount'][0]['CommentCount']
  77.         return num
  78.     except:
  79.         return 'Error'
  80.  
  81. # 获取评论
  82. def getComments(sess,SkuId,url2):
  83.     global comments_num
  84.     res2 = sess.get(url2)
  85.     res2.encoding=res2.apparent_encoding
  86.     json_data=res2.text
  87.     '''
  88.     # 如果用url2_2需要进行如下操作提取json
  89.     start = res2.text.find('jQuery9287224(') + len('jQuery9287224(')
  90.     end = res2.text.find(');')
  91.     json_data=res2.text[start:end]
  92.     '''
  93.     dict_data = json.loads(json_data)
  94.     comments=dict_data['comments']
  95.     for item in comments:
  96.         comment=item['content'].replace('\n','')
  97.         # print(comment)
  98.         comments_num+=1
  99.         try:
  100.             IntoComments(SkuId,comment)
  101.         except Exception as e:
  102.             print(e)
  103.             sess_db.rollback()
  104.  
  105. # 商品信息入库
  106. def IntoGoods(SkuId,title, price, shop, link, comments_num):
  107.     goods_data=Goods(
  108.         sku_id=SkuId,
  109.         name=title,
  110.         price=price,
  111.         comments_num=comments_num,
  112.         shop=shop,
  113.         link=link
  114.     )
  115.     sess_db.add(goods_data)
  116.     sess_db.commit()
  117.  
  118. # 评论入库
  119. def IntoComments(SkuId,comment):
  120.     comments_data=Comments(
  121.         sku_id=SkuId,
  122.         comments=comment
  123.     )
  124.     sess_db.add(comments_data)
  125.     sess_db.commit()
  126.  
  127. if __name__ == '__main__':
  128.     start_time=time.time()
  129.     urls=[]
  130.     KEYWORD=parse.quote(input("请输入要查询的关键词："))
  131.     for i in range(1,2):    # 爬取一页进行测试
  132.         url=f'https://search.jd.com/Search?keyword={KEYWORD}&wq={KEYWORD}&page={i}'
  133.         urls.append(([url,],None))  # threadpool要求必须这样写
  134.     pool=threadpool.ThreadPool(2)  # 2个线程的线程池
  135.     reque=threadpool.makeRequests(getIndex,urls)    # 创建任务
  136.     for r in reque:
  137.         pool.putRequest(r)  # 向线程池提交任务
  138.     pool.wait() # 等待所有任务执行完毕
  139.     print("共获取{}件商品，获得{}条评论，耗时{}".format(num,comments_num,time.time()-start_time))

第三版：

 。。。。不行，又出现空白页了

  1. # 不能爬太快！！！不然获取不到评论
  2. from bs4 import BeautifulSoup
  3. import requests
  4. from urllib import parse
  5. import csv,json,re
  6. import threadpool
  7. import time
  8. from jd_mysqldb import Goods,Comments,sess_db
  9.  
  10. headers={
  11.     'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
  12.     'Cookie': '__jdv=76161171|baidu|-|organic|%25E4%25BA%25AC%25E4%25B8%259C|1613711947911; __jdu=16137119479101182770449; areaId=7; ipLoc-djd=7-458-466-0; PCSYCityID=CN_410000_0_0; shshshfpa=07383463-032f-3f99-9d40-639cb57c6e28-1613711950; shshshfpb=u8S9UvxK66gfIbM1mUNrIOg%3D%3D; user-key=153f6b4d-0704-4e56-82b6-8646f3f0dad4; cn=0; shshshfp=9a88944b34cb0ff3631a0a95907b75eb; __jdc=122270672; 3AB9D23F7A4B3C9B=SEELVNXBPU7OAA3UX5JTKR5LQADM5YFJRKY23Z6HDBU4OT2NWYGX525CKFFVHTRDJ7Q5DJRMRZQIQJOW5GVBY43XVI; jwotest_product=99; __jda=122270672.16137119479101182770449.1613711948.1613738165.1613748918.4; JSESSIONID=C06EC8D2E9384D2628AE22B1A6F9F8FC.s1; shshshsID=ab2ca3143928b1b01f6c5b71a15fcebe_5_1613750374847; __jdb=122270672.5.16137119479101182770449|4.1613748918',
  13.     'Referer': 'https://www.jd.com/'
  14. }
  15.  
  16. num=0   # 商品数量
  17. comments_num=0   # 评论数量
  18.  
  19. # 获取商品信息和SkuId
  20. def getIndex(url):
  21.     global num
  22.     skuids=[]
  23.     session=requests.Session()
  24.     session.headers=headers
  25.     res=session.get(url,headers=headers)
  26.     print(res.status_code)
  27.     res.encoding=res.apparent_encoding
  28.     soup=BeautifulSoup(res.text,'lxml')
  29.     items=soup.select('li.gl-item')
  30.     for item in items[:3]:  # 爬取3个商品测试
  31.         title=item.select_one('.p-name a em').text.strip().replace(' ','')
  32.         price=item.select_one('.p-price strong').text.strip().replace('￥','')
  33.         try:
  34.             shop=item.select_one('.p-shopnum a').text.strip()   # 获取书籍时查找店铺的方法
  35.         except:
  36.             shop=item.select_one('.p-shop a').text.strip()  #   获取其他商品时查找店铺的方法
  37.         link=parse.urljoin('https://',item.select_one('.p-img a').get('href'))
  38.         SkuId=re.search('\d+',link).group()
  39.         skuids.append(([SkuId,session],None))
  40.         headers['Referer'] = f'https://item.jd.com/{SkuId}.html'
  41.         headers['Connection'] = 'keep-alive'
  42.         comments_num=getCommentsNum(SkuId,session)  # 评论数量
  43.         print(SkuId,title, price, shop, link, comments_num)
  44.         print("开始将商品存入数据库...")
  45.         try:
  46.             IntoGoods(SkuId,title, price, shop, link, comments_num)
  47.         except Exception as e:
  48.             print(e)
  49.             sess_db.rollback()
  50.         num += 1
  51.     print("开始获取评论并存入数据库...")
  52.     pool2=threadpool.ThreadPool(3)   # 可同时获取3个商品的评论
  53.     task=threadpool.makeRequests(getComments,skuids)
  54.     for r in task:
  55.         pool2.putRequest(r)
  56.     pool2.wait()
  57.  
  58. # 获取评论
  59. def getComments(SkuId,sess):
  60.     # 获取评论总页数
  61.     url1 = f'https://club.jd.com/comment/productPageComments.action?productId={SkuId}&score=0&sortType=5&page=0&pageSize=10'
  62.     res2 = sess.get(url1, headers=headers)
  63.     res2.encoding = res2.apparent_encoding
  64.     json_data = json.loads(res2.text)
  65.     max_page = json_data['maxPage']  # 经测试最多可获取100页评论，每页10条
  66.     print("{}评论共{}页".format(SkuId, max_page))
  67.     if max_page == 0:
  68.         IntoComments(SkuId, '0')
  69.     else:
  70.         for i in range(0, max_page):
  71.             # 使用此链接获取评论得到的为json格式
  72.             url2 = f'https://club.jd.com/comment/productPageComments.action?productId={SkuId}&score=0&sortType=5&page={i}&pageSize=10'
  73.             # 使用此链接获取评论得到的非json格式，需要提取
  74.             # url2_2=f'https://club.jd.com/comment/productPageComments.action?callback=jQuery9287224&productId={SkuId}&score=0&sortType=5&page={i}&pageSize=10'
  75.             print("开始获取第{}页评论:{}".format(i + 1, url2))
  76.             getComments_one(sess, SkuId, url2)
  77.             time.sleep(1)
  78.  
  79. # 获取评论总数量
  80. def getCommentsNum(SkuId,sess):
  81.     url=f'https://club.jd.com/comment/productCommentSummaries.action?referenceIds={SkuId}'
  82.     res=sess.get(url)
  83.     try:
  84.         res.encoding=res.apparent_encoding
  85.         json_data=json.loads(res.text)  # json格式转为字典
  86.         num=json_data['CommentsCount'][0]['CommentCount']
  87.         return num
  88.     except:
  89.         return 'Error'
  90.  
  91. # 获取单个评论
  92. def getComments_one(sess,SkuId,url2):
  93.     global comments_num
  94.     res2 = sess.get(url2)
  95.     res2.encoding=res2.apparent_encoding
  96.     json_data=res2.text
  97.     '''
  98.     # 如果用url2_2需要进行如下操作提取json
  99.     start = res2.text.find('jQuery9287224(') + len('jQuery9287224(')
  100.     end = res2.text.find(');')
  101.     json_data=res2.text[start:end]
  102.     '''
  103.     dict_data = json.loads(json_data)
  104.     comments=dict_data['comments']
  105.     for item in comments:
  106.         comment=item['content'].replace('\n','')
  107.         # print(comment)
  108.         comments_num+=1
  109.         try:
  110.             IntoComments(SkuId,comment)
  111.         except Exception as e:
  112.             print(e)
  113.             print("rollback！")
  114.             sess_db.rollback()
  115.  
  116. # 商品信息入库
  117. def IntoGoods(SkuId,title, price, shop, link, comments_num):
  118.     goods_data=Goods(
  119.         sku_id=SkuId,
  120.         name=title,
  121.         price=price,
  122.         comments_num=comments_num,
  123.         shop=shop,
  124.         link=link
  125.     )
  126.     sess_db.add(goods_data)
  127.     sess_db.commit()
  128.  
  129. # 评论入库
  130. def IntoComments(SkuId,comment):
  131.     comments_data=Comments(
  132.         sku_id=SkuId,
  133.         comments=comment
  134.     )
  135.     sess_db.add(comments_data)
  136.     sess_db.commit()
  137.  
  138. if __name__ == '__main__':
  139.     start_time=time.time()
  140.     urls=[]
  141.     KEYWORD=parse.quote(input("请输入要查询的关键词："))
  142.     for i in range(1,2):    # 爬取一页进行测试
  143.         url=f'https://search.jd.com/Search?keyword={KEYWORD}&wq={KEYWORD}&page={i}'
  144.         urls.append(([url,],None))  # threadpool要求必须这样写
  145.     pool=threadpool.ThreadPool(2)  # 2个线程的线程池
  146.     reque=threadpool.makeRequests(getIndex,urls)    # 创建任务
  147.     for r in reque:
  148.         pool.putRequest(r)  # 向线程池提交任务
  149.     pool.wait() # 等待所有任务执行完毕
  150.     print("共获取{}件商品，获得{}条评论，耗时{}".format(num,comments_num,time.time()-start_time))

总结：

京东的反爬有点强，如果不想爬取到空白页，只能用单线程加延时一条一条的爬

 

声明：本文内容由网友自发贡献，转载请注明出处：【wpsshop博客】
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号