devbox
2023面试高手
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

python爬取高德地图道路交通状态数据代码_高德路网数据爬取

作者：2023面试高手 | 2024-04-13 21:42:58

赞

踩

高德路网数据爬取
  1.  
  2. """
  3. author: 17839606517
  4. """
  5.  
  6.  
  7. import datetime
  8. import datetime
  9. import os
  10. import csv
  11. from codecs import StreamReaderWriter
  12.  
  13. import numpy as np
  14. import requests
  15. import pandas as pd
  16. import json
  17. import codecs
  18. import time
  19.  
  20.  
  21.  
  22. def aaa():
  23.     # 初始API的URL
  24.     # url="https://restapi.amap.com/v3/traffic/status/rectangle?key=4aa83cdb4436daa9df422838b94b3ea3&extensions=all&rectangle="
  25.     # url = "https://api.map.baidu.com/traffic/v1/polygon?ak="+ak+"&vertexes="
  26.     # url1 = "&coord_type_input=gcj02&coord_type_output=gcj02"
  27.     # 设定整个网格左下角坐标的经纬度值
  28.     # 起点坐标：114.107347,30.475042（汪家嘴立交桥）114.192358,30.528554
  29.     # 起终点坐标改：114.143639,30.477906(东方大道四环线)--114.442451,30.714406
  30.     # 终点坐标：114.432749,30.624948（友谊大道立交）
  31.     # lon:20;lat:8
  32.     ak = ['8oa21GO9kDBUiFCDH9VmEEOS3F7G2UfW', 'GUDOwvKZM8teprBTm1DWBiaq9kqWkmTw',
  33.           'VNpap0WhfCzcQmrmvtjkdrSRUybgGNvL', 'YZdlNgbCUEzItueZ6xZI28LBkik0ZBtN',
  34.           'XLF1DWwGLdN3agDHNFW0GTiKiOTnlCu3', 'FkXcN4asOWY9DWwp1ERA5XZrjIGaLbCG',
  35.           'I69ZxDAtW0hz9N2xc1Cnqi74vPPqzmbZ', 'E8v35ryXDhNgSDiulnNcapRj4EGCNaIV',
  36.           '8oa21GO9kDBUiFCDH9VmEEOS3F7G2UfW', 'YZdlNgbCUEzItueZ6xZI28LBkik0ZBtN',
  37.           'vpVlSRw8f7aHYpbprFk6QBqlomfAMr2A']
  38.  
  39.     akk=0
  40.     baselng = 114.143639
  41.     baselat = 30.477906
  42.     # 设定每个网格单元的经纬度宽
  43.     widthlng = 0.013
  44.     # 同一维度，lng=0.01≈1000米
  45.     widthlat = 0.012
  46.     # 同一经度，lat=0.01≈1113米
  47.     # 用于储存数据
  48.     x=[]
  49.     #用于标识交通态势线段
  50.     num=0
  51.  
  52.     #爬取过程可能会出错中断，因此增加异常处理
  53.     try:
  54.         #循环每个网格进行数据爬取，在这里构建了23X20网格
  55.         for i in range(0,23):
  56.             # print(i)
  57.             xx=i
  58.             #设定网格单元的左下与右上坐标的纬度值
  59.             #在这里对数据进行处理，使之保留6位小数（不保留可能会莫名其妙出错）
  60.             startlat=round(baselat+i*widthlat,6)
  61.             endlat=round(startlat+widthlat,6)
  62.             for j in range(0,20):
  63.                 # print(j)
  64.                 #设定网格单元的左下与右上坐标的经度值
  65.                 startlng=round(baselng+j*widthlng,6)
  66.                 endlng=round(startlng+widthlng,6)
  67.                 #设置API的URL并进行输出测试
  68.                 #a=23*i+20*j
  69.                 # int(a)
  70.                 #b=int(a)
  71.                 akkk=akk%11
  72.                 ak1=ak[akkk]
  73.                 #print(akkk)
  74.                 url = "https://api.map.baidu.com/traffic/v1/polygon?ak=" + ak1 + "&vertexes="
  75.                 url1 = "&coord_type_input=gcj02&coord_type_output=gcj02"
  76.  
  77.                 locStr=str(startlat)+","+str(endlng)+";"+str(endlat)+","+str(endlng)\
  78.                        +";"+str(endlat)+","+str(startlng)+";"+str(startlat)+","+str(startlng)
  79.                 thisUrl=url+locStr+url1
  80.                 #print(thisUrl)
  81.                 #爬取数据
  82.                 data=requests.get(thisUrl)
  83.                 s=data.json()
  84.                 akk=akk+1
  85.                 #print(s)
  86.                 #print(s)
  87.                 #a1=['a','b']
  88.                 a1=s.get('road_traffic')
  89.                 a0=s.get('evaluation')
  90.                 #print(a0.get('status'))
  91.                 #print(type(a1))
  92.                 #print(type(a1) is list)
  93.                 if (type(a1) is list)==False:
  94.                     #print('截至吧')
  95.                     continue
  96.                 #print(a1)
  97.                 #print(i)
  98.                 #print(j)
  99.                 for i1 in range (0,len(a1)):
  100.  
  101.                     #print(len(s.get('description')))
  102.                     if len(s.get('description'))>8:
  103.                         a3 = []
  104.                         a4 = a1[i1].get('congestion_sections')
  105.                         #print(a4==None)
  106.                         if (a4==None)==False:
  107.                             for i2 in range (0,len(a4)):
  108.                                 #print (len(a4))
  109.                                 if len(a4)==1:
  110.                                     a3 = []
  111.                                     curr_time = datetime.datetime.now()
  112.                                     time_str = datetime.datetime.strftime(curr_time, '%Y-%m-%d %H:%M:%S')
  113.                                     a3.append(a1[i1].get('road_name'))
  114.                                     #a3.append(pd.DataFrame([time_str]))
  115.                                     a3.append(time_str)
  116.                                     a3.append(a0.get('status'))
  117.                                     a3.append(a0.get('status_desc'))
  118.                                     a3.append(a4[i2].get('section_desc'))
  119.                                     a3.append(a4[i2].get('status'))
  120.                                     a3.append(a4[i2].get('speed'))
  121.                                     a3.append(a4[i2].get('congestion_distance'))
  122.                                     a3.append(a4[i2].get('congestion_trend'))
  123.                                     f1: StreamReaderWriter = codecs.open("main4.txt", 'a', 'gbk')
  124.                                     #print(a3)
  125.                                     f1.write(str(xx) + ',' + str(j) + ',')
  126.                                     for i in range(0, len(a3)):
  127.                                         f1.write(str(a3[i]) + ",")
  128.                                     f1.write('\n')
  129.                                     f1.close()
  130.                                     a3=[]
  131.                                 else:
  132.                                     a3 = []
  133.                                     curr_time = datetime.datetime.now()
  134.                                     time_str = datetime.datetime.strftime(curr_time, '%Y-%m-%d %H:%M:%S')
  135.                                     a3.append(a1[i1].get('road_name'))
  136.                                     a3.append(time_str)
  137.                                     a3.append(a0.get('status'))
  138.                                     a3.append(a0.get('status_desc'))
  139.                                     a3.append(a4[i2].get('section_desc'))
  140.                                     a3.append(a4[i2].get('status'))
  141.                                     a3.append(a4[i2].get('speed'))
  142.                                     a3.append(a4[i2].get('congestion_distance'))
  143.                                     a3.append(a4[i2].get('congestion_trend'))
  144.                                     f: StreamReaderWriter = codecs.open("main4.txt", 'a','gbk')
  145.                                     #print(a3)
  146.                                     f.write(str(xx) + ',' + str(j) + ',')
  147.                                     for i in range(0, len(a3)):
  148.                                         f.write(str(a3[i]) + ",")
  149.                                     f.write('\n')
  150.                                     f.close()
  151.                                     a3=[]
  152.                                     #print (a4[i2])
  153.                         else:
  154.                             a3 = []
  155.                             #print(len(a1[i1]))
  156.                             curr_time = datetime.datetime.now()
  157.                             time_str = datetime.datetime.strftime(curr_time, '%Y-%m-%d %H:%M:%S')
  158.                             #print(time_str)
  159.                             a3.append(a1[i1].get('road_name'))
  160.                             #a3.append(pd.DataFrame([time_str]))
  161.                             a3.append(time_str)
  162.                             a3.append("1")
  163.                             a3.append("畅通")
  164.                             f2: StreamReaderWriter = codecs.open("main4.txt", 'a','gbk')
  165.                             f2.write(str(xx) + ',' + str(j) + ',')
  166.                             for i in range(0, len(a3)):
  167.                                 f2.write(str(a3[i]) + ",")
  168.                             f2.write('\n')
  169.                             f2.close()
  170.                             a3=[]
  171.  
  172.                     elif len(s.get('description'))==8:
  173.                         a3 = []
  174.                         #print (len(a1[i1]))
  175.                         curr_time = datetime.datetime.now()
  176.                         #print(curr_time)
  177.                         time_str = datetime.datetime.strftime(curr_time, '%Y-%m-%d %H:%M:%S')
  178.                         #print(time_str)
  179.                         a3.append(a1[i1].get('road_name'))
  180.                         a3.append(time_str)
  181.                         a3.append("1")
  182.                         a3.append("畅通")
  183.                         f3: StreamReaderWriter = codecs.open("main4.txt", 'a','gbk')
  184.                         f3.write(str(xx) + ',' + str(j) + ',')
  185.                         for i in range(0, len(a3)):
  186.                             f3.write(str(a3[i]) + ",")
  187.                         f3.write('\n')
  188.                         f3.close()
  189.                         a3=[]
  190.                     else:
  191.                         continue
  192.                    #
  193.                     #print(len(a1[i1]))
  194.                     #print(a3)
  195.                     '''a3= np.asarray(a3)
  196.                     np.savetxt("temp.csv",a3,delimiter=",")'''
  197.                     '''if not os.path.exists('result_s1.csv'):
  198.                         a3.to_csv('result_s1.csv', encoding='gbk', mode='a', index=False, index_label=False)  # 保存数据
  199.                     else:
  200.                         a3.to_csv('result_s1.csv', encoding='gbk', mode='a', index=False, index_label=False,header=False)'''
  201.                     '''with open("test.txt","w")as csvfile:
  202.                         for i in range(0,len(a3)):
  203.                         writer=csv.writer(csvfile)
  204.                         #writer.writerow(['1','2','3','4','5','6','7','8','9'])
  205.                         writer.writerows(a3)'''
  206.  
  207.                 #a=s.get('road_traffic').get('road_name')
  208.                 #print(a)
  209.                 '''
  210.                 print(a.get('description'))
  211.                 #注意，提取数值需要使用XXX.get()的方式来实现，如a[k].get('speed')
  212.                 #若使用a[k]['speed']来提取，或会导致KeyError错误
  213.                 for k in range(0,len(a)):
  214.                     #if s['description']!='该区域整体畅通。'
  215.                     s2=a[k]['congestion_distance']
  216.                         #s3=s2.split(";")
  217.                     for l in range(0,len(s2)):
  218.                         s4=x.append(s2) 
  219.                         x.append([a[k].get('section_desc'),a[k].get('status'),a[k].get('speed'),num,float(s4)])
  220.                         print(x)
  221.                     num=num+1
  222.                 '''
  223.                 #time.sleep(0.1)
  224.                 #若爬取网格较多，可使用time.sleep(秒数)来避免高德的单秒API调用次数的限制
  225.     except Exception  as e:
  226.         pass
  227. var=1
  228. while var==1:
  229.     ts=time.time()
  230.     start_time = datetime.datetime.strptime(str(datetime.datetime.now().date()) + '04:59', '%Y-%m-%d%H:%M')
  231.     end_time = datetime.datetime.strptime(str(datetime.datetime.now().date()) + '23:59', '%Y-%m-%d%H:%M')
  232.     now_time = datetime.datetime.now()
  233.     if now_time > start_time and now_time < end_time:
  234.         aaa()
  235.     else:
  236.         continue
  237.     ts1=time.time()
  238.     time.sleep(3600-ts1+ts)
  239. #将数据结构化存储至规定目录的CSV文件中
  240. '''
  241. c = pd.DataFrame(x,columns=['section_desc','status','speed','congestion_distance','x','y'])
  242. c.to_csv('E:/新建文件夹/600/爬虫.csv',encoding='utf-8-sig')
  243. '''

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/2023面试高手/article/detail/418689
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号