python爬虫爬取音乐单曲_使用Python爬虫爬取招聘网站

# 导入相关的包
import requests
import time
import csv
from urllib.parse import urlencode
from urllib.parse import quote
 
# 使用面向对象的方式
class Spider:
    def __init__(self, page, cityId, search_keywords):
        '''初始化方法'''
        start = (page - 1) * 90  # 搜索结果每页90个，请求参数的start的值为0,90,180等。
        self.headers = {
            'referer': 'https://sou.zhaopin.com/?jl=' + str(cityId) + '&kw=' + quote(search_keywords) + '&kt=3',
            'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko)',
            'x-requested-with': 'XMLHttpRequest'  # ajax异步请求
            }
        self.params = {
             'start': start,
             'pageSize': '90',
             'cityId': cityId,
             'workExperience': -1,
             'education': -1,
             'companyType': -1,
             'employmentType': -1,
             'jobWelfareTag': -1,
             'kw': search_keywords,
             'kt': 3,
             '_v': '0.00882343',
             'x-zp-page-request-id': '',
             'x-zp-client-id': ''
            }
        self.url_search = 'https://fe-api.zhaopin.com/c/i/sou?' + urlencode(self.params)
 
    def get_one_page(self):
        '''请求网页'''
        try:
            response = requests.get(self.url_search, headers=self.headers)
            if response.status_code == 200:
                return response.json()
        except requests.ConnectionError:
            print('连接错误')
            return None
 
    def parse_one_page(self, response_json):
        '''解析网页'''
        if response_json:
            items = response_json.get('data').get('results')
            for item in items:
                crawlTime = str(time.ctime())  # 抓取时间
                businessArea = item.get('businessArea')  # 公司所在区域
                city = item.get('city').get('items')[0].get('name')  # 公司所在城市
                companyName = item.get('company').get('name')  # 公司名称
                companyNumber = item.get('company').get('number')  # 公司ID
                companySize = item.get('company').get('size').get('name')  # 公司人数规模
                eduLevel = item.get('eduLevel').get('name')  # 职位要求的学历
                jobName = item.get('jobName')  # 职位名称
                jobNumber = item.get('number')  # 职位ID
                jobType = item.get('jobType').get('items')[0].get('name')  # 职位类别
                positionURL = item.get('positionURL')  # 职位网址
                salary = item.get('salary')  # 薪资
                updateDate = item.get('updateDate')  # 职位更新时间
                workingExp = item.get('workingExp').get('name')  # 工作年限要求
                zhilian_results = [crawlTime, businessArea, city, companyName, companyNumber, companySize, eduLevel,
                                  jobName, jobNumber, jobType, positionURL, salary, updateDate, workingExp]
                print('zhilian_results:', zhilian_results)
                yield zhilian_results
 
    def save_to_csv(self, zhilian_results):
        '''保存数据到CSV文件'''
        headers = ['crawlTime', 'businessArea', 'city', 'companyName', 'companyNumber',
                   'companySize', 'eduLevel', 'jobName', 'jobNumber', 'jobType', 'positionURL',
                   'salary'  , 'updateDate', 'workingExp']
        with open('zhilian_results.csv', 'a', encoding='utf-8', newline='') as f:
            f_csv = csv.writer(f)
            f_csv.writerow(headers)
            f_csv.writerows(zhilian_results)
 
    def run(self):
        '''启动函数'''
        response_json = self.get_one_page()
        zhilian_search_results = self.parse_one_page(response_json)
        self.save_to_csv(zhilian_search_results)
 
if __name__ == '__main__':
    # 抓取搜索相关性较高的前3页
    for i in range(1, 4):
        time.sleep(1)
        s = Spider(i, 763, '数据分析')
        s.run()