pyhton爬虫（12）——抓取携程酒店评论数据_python 爬取 携程酒店价格

本文主要目标是抓取携程酒店基本信息和用户评论数据。具体来说，酒店基本信息包括：酒店名、酒店星级、酒店最低房价、用户推荐比、酒店总评分等；用户评论数据包括：用户评论时间，用户评分，评论内容等。

实现代码如下所示：

# -*- coding: utf-8 -*-
"""
Created on Mon Aug  7 21:05:03 2017

@author: Administrator
"""

import urllib.request
from bs4 import BeautifulSoup
import pymysql.cursors

'''
目标:获取酒店名称和酒店星级
'''
url1 = 'http://hotels.ctrip.com/hotel/435383.html'

html1 = urllib.request.urlopen(url1).read().decode('utf-8')

#1.获取酒店名称信息
soup1 = BeautifulSoup(html1,'lxml')

result1 = soup1.find_all(attrs={"itemprop":"name"})
hotelName = result1[0].string
print("酒店名称为:{}".format(hotelName))

#2.获取酒店星级信息
soup12 = BeautifulSoup(html1,'lxml')

result12 = soup12.find_all(attrs={"class":"grade"})

#print(result1)
result12 =str(result12)

soup13 = BeautifulSoup(result12,'lxml')

result13 = soup13.find_all('span')
hotelStar = result13[0]['title']
print("酒店星级为:{}".format(hotelStar))

'''
目标:获取酒店最低房价和评论总数
'''
url2 = "http://m.ctrip.com/html5/hotel/HotelDetail/435383.html"

html2 = urllib.request.urlopen(url2).read().decode('utf-8')

#获取酒店最低价
soup2 = BeautifulSoup(html2,'lxml')

result2 = soup2.find_all(attrs={"class":"js-cas-p"})
lowPrice = result2[0]['data-cas-p']
print("酒店最低房价为:{}".format(lowPrice))

#评论总数
result21 = soup2.find_all(attrs={"class":"dt-color12 dt-fn15"})
commentCounts =  result21[0].string
print("评论总数为:{}".format(commentCounts))

'''
目标:获取酒店卫生评分、环境评分、服务评分、设施评分、用户推荐比、用户评分、评价内容
'''

url3 = 'http://m.ctrip.com/html5/hotel/HotelDetail/dianping/435383.html'

html3 = urllib.request.urlopen(url3).read().decode('utf-8')

soup3 = BeautifulSoup(html3,'lxml')

#获取酒店各项评分数据
result32 = soup3.find_all(attrs={"class":"ve-txt"})
result32 = str(result32)
soup32 = BeautifulSoup(result32,'lxml')

result33 = soup32.find_all('em')
userRecommendRate = result33[0].string
hRating = result33[1].string
eRating = result33[2].string
sRating = result33[3].string
iRating = result33[4].string

print("用户推荐为:{}".format(userRecommendRate))
print("卫生评分为:{}分".format(hRating))
print("环境评分为:{}分".format(eRating))
print("服务评分为:{}分".format(sRating))
print("设施评分为:{}分".format(iRating))

#提取用户评论数据
result34 = soup3.find_all(attrs={"class":"hotel-cell-num"})
result34 = str(result34[1])
soup33 = BeautifulSoup(result34,'lxml')
result35 = soup33.find_all('p')

for i in range(0,10):
    #userName = result35[i].get_text()
    result36 = soup3.find_all(attrs={"itemprop":"datePublished"})
    datePublished = result36[i].string
    print("评论发表于:{}".format(datePublished))

    result37 = soup3.find_all(attrs={"itemprop":"ratingValue"})
    userRating = result37[i].string
    print("评分为:{}".format(userRating))

    result38 = soup3.find_all(attrs={"class":"tree-ellips-line6 js_arr"})
    commentText = result38[i].get_text()
    print("评论内容为:{}".format(commentText))

    '''
        数据库操作
    '''

    #获取数据库链接
    connection  = pymysql.connect(host = 'localhost',
                              user = 'root',
                              password = '123456',
                              db = 'xiecheng',
                              charset = 'utf8mb4')
    try:
        #获取会话指针
        with connection.cursor() as cursor:
            #创建sql语句
            sql = "insert into `macro-polo` (`hotelName`,`hotelStar`,`lowPrice`,`commentCounts`,`userRecommendRate`,`hRating`,`eRating`,`sRating`,`iRating`,`datePublished`,`userRating`,`commentText`) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"

            #执行sql语句
            cursor.execute(sql,(hotelName,hotelStar,lowPrice,commentCounts,userRecommendRate,hRating,eRating,sRating,iRating,datePublished,userRating,commentText))

            #提交数据库
            connection.commit()
    finally:
        connection.close()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

运行结果如下下图所示：

MySQL数据库中的数据存储如下图所示：