python简单爬取淘宝商品信息_python爬取淘宝店铺信息数据

爬取淘热卖商品“泡面”的信息，包括商品名称，店铺，链接，付款人数，价格等，用csv保存

import requests
import csv
import time
import parsel

def get_url(url):
    headers={
        'user-agent': '自己的user-agent',
        'cookie': '自己的cookir',

    }
    response=requests.get(url=url,headers=headers,timeout=3)
    print('调用get')
    if response.status_code==200:
        return response.text
    else:
        return None

def parse_url():
    print('调用parse')
    htmltext=get_url(url)
    selector=parsel.Selector(htmltext)
    content=selector.css('.item')
    print(type(content))
    global dit
    for li in content[0:-7]:
        try:
            name1=li.css('.info')
            name2=name1.css('span')
            name=name2.css('.title::text').get()
            if name:
                store=li.css('.shopNick::text').get()
                #location=li.css('::text').get()
                price=li.css('strong::text').get()
                num=li.css('.payNum::text').get().replace('人付款','')
                href=li.css('a::attr(href)').get()
            else:
                print('NONE')
        except AttributeError as e:
            print(e)

        dit={
            '商品名称':name,
            '店铺':store,
            #'位置':location,
            '商品价格':price,
            '付款人数':num,
            '商品链接':href
        }
        print(dit)
        save()

def save():
    csv_writer.writerow(dit)
    print('调用save')
def main(i):
    global url
    url='https://re.taobao.com/search?refpid=420435_1006&keyword=%E6%B3%A1%E9%9D%A2&_input_charset=utf-8&page={a}&isinner=0'
    parse_url()


if __name__=='__main__':
    f=open('D://taobaopaomian.csv','a+',encoding='utf-8',newline='')
    csv_writer=csv.DictWriter(f,fieldnames=['商品名称','店铺','商品价格','付款人数','商品链接'])
    csv_writer.writeheader()
    for a in range(3):
        main(a)
        time.sleep(2)
    print('下载完成')
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69