基于医疗知识图谱的项目构建学习总结（一）—项目构建环境搭建及爬取数据部分_眼科的知识图谱构建数据

由于工作需要，这里学习了中科院软件所刘焕勇老师在github上的开源项目，基于知识图谱的医药领域问答项目QABasedOnMedicaKnowledgeGraph。
原项目地址：https://github.com/liuhuanyong/QASystemOnMedicalKG
自己动手实现了环境的搭建，目前实践到爬虫部分，在此记录，欢迎大家提出意见。

首先是安装mongodb

参考博客mongodb安装及创建用户
按照文中的说明下载和配置mongodb，并启动服务，打开网址，出现如下语句说明启动成功：

It looks like you are trying to access MongoDB over HTTP on the native driver port.
1

为了便于验证数据库是否建立成功，这里给出了几个常用的数据库语法：

show dbs 查看已有的数据库
use db_name 如果该数据库存在则进入，若不存在则创建名称为db_name的数据库
db.dropDatabase() 删除该数据库
db.jc.find() 查看数据库中jc表中的数据

然后是爬取数据部分

在对症状的解析函数中，源代码对有的网页解析得到的结果是人名，因此我对其进行了修改，详见代码。运行之后就在数据库中建立该数据库，并存入了爬取的数据。

# -*- coding: utf-8 -*-
import urllib.request
import urllib.parse
from lxml import etree
import pymongo


# 创建数据库连接，开启Mongodb服务之后，不存在该数据库的话会自动创建该数据库
conn = pymongo.MongoClient()
db = conn['medical']
col = db['data']


# 爬取数据并解析
def get_html(url):
    headers = {'User-Agent':'Mozilla/5.0(Window Nt 10.0; WOW64) AppleWebKit/537.36 (KHTML, LIKE Gecko)'}
    req = urllib.request.Request(url=url, headers=headers)
    res = urllib.request.urlopen(req)
    html = res.read().decode('gbk')
    return html


def spider_main():
    for page in range(1, 11000):
        try:
            basic_url = 'http://jib.xywy.com/il_sii/gaishu/%s.htm'% page
            cause_url = 'http://jib.xywy.com/il_sii/cause/%s.htm' % page  # 病因
            prevent_url = 'http://jib.xywy.com/il_sii/prevent/%s.htm' % page  # 预防
            symptom_url = 'http://jib.xywy.com/il_sii/symptom/%s.htm' % page  # 症状
            inspect_url = 'http://jib.xywy.com/il_sii/inspect/%s.htm' % page  # 检查方法
            treat_url = 'http://jib.xywy.com/il_sii/treat/%s.htm' % page  # 治疗
            food_url = 'http://jib.xywy.com/il_sii/food/%s.htm' % page  # 饮食保健
            drug_url = 'http://jib.xywy.com/il_sii/drug/%s.htm' % page  # 好评药品
            data = {}
            data['url'] = basic_url
            data['basic_info'] = basicinfo_spider(basic_url)
            data['cause_info'] = common_spider(cause_url)
            data['prevent_info'] = common_spider(prevent_url)
            data['symptom'] = symptom_spider(symptom_url)
            data['inspect_info'] = inspect_spider(inspect_url)
            data['treat_info'] = treat_sipder(treat_url)
            data['food_info'] = food_spider(food_url)
            data['drug_info'] = drug_spider(drug_url)
            print(page, basic_url)
            col.insert(data)
        except:
            print('error')
    return


'''基本信息解析'''
def basicinfo_spider(url):
    html = get_html(url)
    selector = etree.HTML(html)
    title = selector.xpath('//title/text()')[0]
    category = selector.xpath('//div[@class="wrap mt10 nav-bar"]/a/text()')
    desc = selector.xpath('//div[@class="jib-articl-con jib-lh-articl"]/p/text()')
    ps = selector.xpath('//div[@class="mt20 articl-know"]/p')
    infobox = []
    for p in ps:
        info = p.xpath('string(.)').replace('\r','').replace('\n','').replace('\xa0', '').replace('   ', '').replace('\t','')
        infobox.append(info)
    basic_data = {}
    basic_data['category'] = category
    basic_data['name'] = title.split('的简介')[0]
    basic_data['desc'] = desc
    basic_data['attributes'] = infobox
    return basic_data

# 对网页分别进行解析

def treat_sipder(url):
    html = get_html(url)
    selector = etree.HTML(html)
    ps = selector.xpath('//div[starts-with(@class, "mt20 articl-know")]/p')
    infobox = []
    for p in ps:
        info = p.xpath('string(.)').replace('\r','').replace('\n','').replace('\xa0','').replace('   ','').replace('\t','')
        infobox.append(info)
    # print(infobox)
    return infobox

def drug_spider(url):
    html = get_html(url)
    selector = etree.HTML(html)
    drugs = [i.replace('\n','').replace('\t', '').replace(' ','') for i in
             selector.xpath('//div[@class="fl drug-pic-rec mr30"]/p/a/text()')]
    # print(drugs)
    return drugs


'''food治疗解析'''
def food_spider(url):
    html = get_html(url)
    selector = etree.HTML(html)
    divs = selector.xpath('//div[@class="diet-img clearfix mt20"]')
    try:
        food_data = {}
        food_data['good'] = divs[0].xpath('./div/p/text()')
        food_data['bad'] = divs[1].xpath('./div/p/text()')
        food_data['recommand'] = divs[2].xpath('./div/p/text()')
    except:
        return {}
    return food_data


'''症状信息解析'''
def symptom_spider(url):
    html = get_html(url)
    selector = etree.HTML(html)
    
    # 源代码中的结果是人名，因此对其进行了修改。
    symptoms = selector.xpath('//span[@class="db f12 lh240 mb15 "]/a/text()')   
    ps = selector.xpath('//p')
    detail = []
    for p in ps:
        info = p.xpath('string(.)').replace('\r','').replace('\n','').replace('\xa0', '').replace('   ', '').replace('\t','')
        detail.append(info)
    symptoms_data = {}
    symptoms_data['symptoms'] = symptoms
    symptoms_data['symptoms_detail'] = detail
    # print(symptoms_data)
    return symptoms, detail

def inspect_spider(url):
    '''对检查项目的链接进行获取，有的网页时有检查链接的，有的是没有的'''
    html = get_html(url)
    selector = etree.HTML(html)
    inspects = selector.xpath('//li[@class="check-item"]/a/@href')
    return inspects


'''通用解析模块'''
def common_spider(url):
    html = get_html(url)
    selector = etree.HTML(html)
    ps = selector.xpath('//p')
    infobox = []
    for p in ps:
        info = p.xpath('string(.)').replace('\r', '').replace('\n', '').replace('\xa0', '').replace('   ','').replace('\t', '')
        if info:
            infobox.append(info)

    return '\n'.join(infobox)


'''检查项抓取模块'''
def inspect_crawl():
    for page in range(1, 3685):
        try:
            url = 'http://jck.xywy.com/jc_%s.html'%page
            html = get_html(url)
            data = {}
            data['url'] = url
            data['html'] = html
            db['jc'].insert(data)
            # print(data)
        except Exception as e:
            print(e)
spider_main()
inspect_crawl()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161

接着将解析的数据存入数据库中

1. 在这一部分代码中，需要将max_cut中的disease.txt位置进行修改为…/dict/disease.txt，还需要在这个打开文件的时候加入encoding=‘utf-8’，即open(dict_path, encoding=‘utf-8’)，不然会报错。
2. 在下边的导入代码中有first_name.txt文件，在源文件中并没有给出，我猜想应该是这个症状描述不包含疾病名称，因此我使用了disease.txt文件代替的。
3. 执行程序的时候，必须要先执行这个函数modify_jc()，更新jc表的值，这样在执行函数collect_medical()的时候才不会报name错误。

#!/usr/bin/env python3
# coding: utf-8
# File: build_data.py
# Author: lhy<lhy_in_blcu@126.com,https://huangyong.github.io>
# Date: 18-10-3
import pymongo
from lxml import etree
import os
from max_cut import *

class MedicalGraph:
    def __init__(self):
        self.conn = pymongo.MongoClient()
        # cur_dir = '/'.join(os.path.abspath(__file__).split('/')[:-1])
        self.db = self.conn['test']
        self.col = self.db['data']
        first_words = [i.strip() for i in open('../dict/disease.txt', encoding='utf-8')]
        alphabets = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y', 'z']
        nums = ['1','2','3','4','5','6','7','8','9','0']
        self.stop_words = first_words + alphabets + nums
        self.key_dict = {
            '医保疾病' : 'yibao_status',
            "患病比例" : "get_prob",
            "易感人群" : "easy_get",
            "传染方式" : "get_way",
            "就诊科室" : "cure_department",
            "治疗方式" : "cure_way",
            "治疗周期" : "cure_lasttime",
            "治愈率" : "cured_prob",
            '药品明细': 'drug_detail',
            '药品推荐': 'recommand_drug',
            '推荐': 'recommand_eat',
            '忌食': 'not_eat',
            '宜食': 'do_eat',
            '症状': 'symptom',
            '检查': 'check',
            '成因': 'cause',
            '预防措施': 'prevent',
            '所属类别': 'category',
            '简介': 'desc',
            '名称': 'name',
            '常用药品' : 'common_drug',
            '治疗费用': 'cost_money',
            '并发症': 'acompany'
        }
        self.cuter = CutWords()

    def collect_medical(self):
        cates = []
        inspects = []
        count = 0
        for item in self.col.find():
            data = {}
            basic_info = item['basic_info']
            name = basic_info['name']
            if not name:
                continue
            # 基本信息
            data['名称'] = name
            data['简介'] = '\n'.join(basic_info['desc']).replace('\r\n\t', '').replace('\r\n\n\n','').replace(' ','').replace('\r\n','\n')
            category = basic_info['category']
            data['所属类别'] = category
            cates += category
            inspect = item['inspect_info']
            inspects += inspect
            attributes = basic_info['attributes']
            # 成因及预防
            data['预防措施'] = item['prevent_info']
            data['成因'] = item['cause_info']
            # 并发症
            data['症状'] = list(set([i for i in item["symptom_info"][0] if i[0] not in self.stop_words]))
            for attr in attributes:
                attr_pair = attr.split('：')
                if len(attr_pair) == 2:
                    key = attr_pair[0]
                    value = attr_pair[1]
                    data[key] = value
            # 检查
            inspects = item['inspect_info']
            jcs = []
            for inspect in inspects:
                jc_name = self.get_inspect(inspect)
                if jc_name:
                    jcs.append(jc_name)
            data['检查'] = jcs
            # 食物
            food_info = item['food_info']
            if food_info:
                data['宜食'] = food_info['good']
                data['忌食'] = food_info['bad']
                data['推荐'] = food_info['recommand']
            # 药品
            drug_info = item['drug_info']
            data['药品推荐'] = list(set([i.split('(')[-1].replace(')','') for i in drug_info]))
            data['药品明细'] = drug_info
            data_modify = {}
            for attr, value in data.items():
                attr_en = self.key_dict.get(attr)
                if attr_en:
                    data_modify[attr_en] = value
                if attr_en in ['yibao_status', 'get_prob', 'easy_get', 'get_way', "cure_lasttime", "cured_prob"]:
                    data_modify[attr_en] = value.replace(' ','').replace('\t','')
                elif attr_en in ['cure_department', 'cure_way', 'common_drug']:
                    data_modify[attr_en] = [i for i in value.split(' ') if i]
                elif attr_en in ['acompany']:
                    acompany = [i for i in self.cuter.max_biward_cut(data_modify[attr_en]) if len(i) > 1]
                    data_modify[attr_en] = acompany

            try:
                self.db['medical'].insert(data_modify)
                count += 1
                print(count)
            except Exception as e:
                print(e)

        return


    def get_inspect(self, url):
        res = self.db['jc'].find_one({'url':url})
        if not res:
            return ''
        else:
            return res['name']

    def modify_jc(self):
        for item in self.db['jc'].find():
            url = item['url']
            content = item['html']
            selector = etree.HTML(content)
            name = selector.xpath('//title/text()')[0].split('结果分析')[0]
            desc = selector.xpath('//meta[@name="description"]/@content')[0].replace('\r\n\t','')
            self.db['jc'].update({'url':url}, {'$set':{'name':name, 'desc':desc}})


if __name__ == '__main__':
    handler = MedicalGraph()
    handler.modify_jc()
    handler.collect_medical()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140

完成之后，使用如下命令可以导出数据

mongoexport -d test -c medical -o medical.json
1

其中，-d test 是指明数据库,
-c medical 是指明要导出的列表
-o medical.json 是指明导出的文件名，可以在前边指定位置