scrapy_redis分布式爬虫 从redis写到mysql数据库中_scrapy-redis爬虫redis缓存数据乱码

import redis
import MySQLdb
import json

def process_item():
# 创建redis数据库连接
rediscli = redis.Redis(host = “127.0.0.1”, port = 6379, db = 0)

# 创建mysql数据库连接
mysqlcli = MySQLdb.connect(host = "127.0.0.1", port = 3306, \
    user = "", passwd = "", db = "数据库")

offset = 0

while True:
    # 将数据从redis里pop出来
    source, data = rediscli.blpop("表名:items")
    item = json.loads(data)
    try:
        # 创建mysql 操作游标对象，可以执行mysql语句
        cursor = mysqlcli.cursor()

        cursor.execute("insert into 表名 (username, age) values (%s, %s)", [item['username'], item['age']])
        # 提交事务
        mysqlcli.commit()
        # 关闭游标
        cursor.close()
        offset += 1
        print offset
    except:
        pass
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

if name == “main“:
process_item()