热门标签
热门文章
- 1Air724UG 核心板学习记录-开发板介绍
- 2看大神如何用python爬虫爬取京东商品评论_python爬虫爬取京东电商平台商品评论,要求根据商品关键字爬取
- 3探索【Stable-Diffusion WEBUI】的附加功能：图片缩放&抠图_4x-ultrasharp
- 4AI程序员的出现与程序员的未来：饭碗之争的真相
- 548位一作相聚CVPR 2024预讲会/附全部议程
- 6Flask之ajax操作示例_flask 处理 ajex
- 7LLaMA模型指令微调 字节跳动多模态视频大模型 Valley 论文详解_valley: video assistant with large language model
- 8（五）比赛中的CV算法（下）目标检测终章：Vision Transformer_为什么不用vision transformer 做bacbone 进行目标检测
- 9包含密钥的OMP压缩感知模拟（MATLAB）
- 10备战蓝桥杯，用JAVA刷洛谷算法题单：【算法2-1】前缀和、差分与离散化
当前位置: article > 正文
医药问答系统（一）python将excel或json数据处理后存入neo4j_excel neo4j
作者：AllinToyou | 2024-04-05 04:56:34
赞
踩
excel neo4j
1.将excel的数据存入neo4j
本程序使用Jupyter运行，若使用pycharm还需安装py2neo
pip install py2neo==5.0b1 注意版本，要不对应不了
文档：https://py2neo.org/v4/index.html
- 在第一个文件中
- 用panda读取excel中的数据
- 将取出的数据中的实体去重，另外保存到变量中，最终用于创建结点：例中data_extraction()
- 将读取到的数据抽取需要的列，再次拼成dataframe，最终用于创建关系：例中relation_extraction()
- 在第二个文件中
- 通过py2neo工具将这些结点和关系导入neo4j
# -*- coding: utf-8 -*-
import os
import pandas as pd
invoice_data = pd.read_excel('D:/BaiduNetdiskDownload/pandasDemo/Invoice_data_Demo.xls', header=0)
#print("取到的数据：", invoice_data)
#从原数据中将需要创建的实体(买方、卖方)节点抽取出来，将所有的数据全部保存到数组中
def data_extraction():
"""节点数据抽取"""
# 取出所有买方名称到node_buy_key[]
node_buy_key = []
for i in range(0, len(invoice_data)):
node_buy_key.append(invoice_data['购买方名称'][i])
node_sell_key = []
for i in range(0, len(invoice_data)):
node_sell_key.append(invoice_data['销售方名称'][i])
# 去除重复的买方/卖方名称
node_buy_key = list(set(node_buy_key))
node_sell_key = list(set(node_sell_key))
#除了第一列，将所有数据按行取出存到node_list_value[]
node_list_value = []
for i in range(0, len(invoice_data)):
for n in range(1, len(invoice_data.columns)):
node_list_value.append(invoice_data[invoice_data.columns[n]][i])
# set()去重,list()转化成列表
node_list_value = list(set(node_list_value))
# 将list中浮点及整数类型全部转成string类型
node_list_value = [str(i) for i in node_list_value]
#返回所有去重后的购买方名称，去重后的销售方名称，以及所有数据
return node_buy_key, node_sell_key,node_list_value
#将原数据中需要用到的列抽取出来，并且再次拼成excel的样子
def relation_extraction():
"""联系数据抽取"""
links_dict = {}
sell_list = [] # 销售方列表
money_list = [] # 交易额列表
buy_list = [] # 购买方列表
# 取列名--“金额”
# print("*****", invoice_data.columns[19], "********")
for i in range(0, len(invoice_data)):
money_list.append(invoice_data[invoice_data.columns[19]][i])# 将所有金额依次导入
sell_list.append(invoice_data[invoice_data.columns[10]][i])# 将所有销售方依次导入
buy_list.append(invoice_data[invoice_data.columns[6]][i])# 将所有购买方依次导入
# 将数据中int类型全部转成string
sell_list = [str(i) for i in sell_list]
buy_list = [str(i) for i in buy_list]
money_list = [str(i) for i in money_list]
# 整合数据，将三个list整合成一个dict，字典里面存储了多个数组的首地址
links_dict['buy'] = buy_list
links_dict['money'] = money_list
links_dict['sell'] = sell_list
# 将数据转成DataFrame---类似excel的格式
df_data = pd.DataFrame(links_dict)
return df_data
# 实例化
create_data = DataToNeo4j()
# 调用create_data对象的方法创建结点,传参时调用本文件的data_extraction方法
create_data.create_node(data_extraction()[0], data_extraction()[1])
create_data.create_relation(relation_extraction())
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
- 72
- 73
- 74
- 75
- 76
- 77
- 78
- 79
# -*- coding: utf-8 -*-
from py2neo import Node, Graph, Relationship,NodeMatcher
"""将实体列表和关系dataframe存入neo4j"""
class DataToNeo4j(object):
def __init__(self):
"""建立连接"""
link = Graph("http://localhost:7474", username="neo4j", password="123456")
self.graph = link
# 定义label即节点类型
self.buy = 'buy'
self.sell = 'sell'
self.graph.delete_all()
self.matcher = NodeMatcher(link)
# 三引号是注释，官方小例子,帮助理解
#Node()定义结点，Relationship()定义关系，create()创建结点或关系
"""
node3 = Node('animal' , name = 'cat')
node4 = Node('animal' , name = 'dog')
node2 = Node('Person' , name = 'Alice')
node1 = Node('Person' , name = 'Bob')
r1 = Relationship(node2 , 'know' , node1)
r2 = Relationship(node1 , 'know' , node3)
r3 = Relationship(node2 , 'has' , node3)
r4 = Relationship(node4 , 'has' , node2)
self.graph.create(node1)
self.graph.create(node2)
self.graph.create(node3)
self.graph.create(node4)
self.graph.create(r1)
self.graph.create(r2)
self.graph.create(r3)
self.graph.create(r4)
"""
def create_node(self, node_buy_key,node_sell_key):
"""建立节点"""
for name in node_buy_key:
buy_node = Node(self.buy, name=name)
self.graph.create(buy_node)
for name in node_sell_key:
sell_node = Node(self.sell, name=name)
self.graph.create(sell_node)
def create_relation(self, df_data):
"""建立联系"""
m = 0
for m in range(0, len(df_data)):
try:
rel = Relationship(self.matcher.match(self.buy).where("_.name=" + "'" + df_data['buy'][m] + "'").first(),
df_data['money'][m], self.matcher.match(self.sell).where("_.name=" + "'" + df_data['sell'][m] + "'").first())
self.graph.create(rel)
except AttributeError as e:
print(e, m)
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
Invoice_data_Demo.xls
链接：https://pan.baidu.com/s/1lgSg47YWW-6sfC1T5D5P2w
提取码：uxg6
2.将json数据存入neo4j
本程序使用Jupyter运行
- 读取json文件
- 自定义函数read_nodes()通过循环判断
- 如果是疾病的属性，记录在相应的列表中
- 如果是可提取为实体的属性，不仅要记录在相应的列表中，还要记录实体间关系
- 每一轮循环将所有属性拼接成一个字典，存入disease_infos[]
- 自定义函数create_diseases_nodes(disease_infos[])生成疾病类结点：Node()定义结点；create()在neo4j中创建结点
- 自定义函数create_node()生成其他所有类型的结点：Node()定义结点；create()在neo4j中创建结点
- 自定义函数create_relationship()创建关系：利用sql语句查找两结点并创建关系
- 将关系列表中的两个实体中间用“###”隔开，再用set()去重
- 将去重后的列表用split分开保存在两个变量中，拼接出sql语句
- 用run()执行sql语句
#选材自开源项目(刘焕勇，中国科学院软件研究所)，数据集来自互联网爬虫数据
import os
import json
from py2neo import Graph,Node
class MedicalGraph:
def __init__(self):
cur_dir = 'D:\\BaiduNetdiskDownload\\QAMedicalKG'
self.data_path = os.path.join(cur_dir, 'data/medical2.json')
#建立连接
self.g = Graph("http://localhost:7474", username="neo4j", password="123456")
'''读取文件：分别记录所有结点，关系，以及疾病全部信息的字典'''
def read_nodes(self):
# 用于记录结点：共７类节点
drugs = [] # 药品
foods = [] # 食物
checks = [] # 检查
departments = [] #科室
producers = [] #药品大类
diseases = [] #记录疾病的名称
symptoms = []#症状
# 用于记录实体间关系
rels_department = [] # 科室－科室关系
rels_noteat = [] # 疾病－忌吃食物关系
rels_doeat = [] # 疾病－宜吃食物关系
rels_recommandeat = [] # 疾病－推荐吃食物关系
rels_commonddrug = [] # 疾病－通用药品关系
rels_recommanddrug = [] # 疾病－热门药品关系
rels_check = [] # 疾病－检查关系
rels_drug_producer = [] # 厂商－药物关系
rels_symptom = [] #疾病症状关系
rels_acompany = [] # 疾病并发关系
rels_category = [] # 疾病与科室之间的关系
#用于记录疾病的所有信息
disease_infos = []
count = 0
for data in open(self.data_path):
disease_dict = {}
count += 1
#读取json文件数据
data_json = json.loads(data)
#将 JSON 对象转换为 Python 字典
disease = data_json['name']
diseases.append(disease)
#这部分用于记录疾病的属性
disease_dict['name'] = disease
disease_dict['desc'] = ''
disease_dict['prevent'] = ''
disease_dict['cause'] = ''
disease_dict['easy_get'] = ''
disease_dict['cure_department'] = ''
disease_dict['cure_way'] = ''
disease_dict['cure_lasttime'] = ''
disease_dict['symptom'] = ''
disease_dict['cured_prob'] = ''
#如果该条json记录中存在下述字段：记录字段、关系
if 'symptom' in data_json:
symptoms += data_json['symptom']#记录字段
for symptom in data_json['symptom']:
rels_symptom.append([disease, symptom])#记录关系
if 'acompany' in data_json:
for acompany in data_json['acompany']:
rels_acompany.append([disease, acompany])
if 'desc' in data_json:
disease_dict['desc'] = data_json['desc']
if 'prevent' in data_json:
disease_dict['prevent'] = data_json['prevent']
if 'cause' in data_json:
disease_dict['cause'] = data_json['cause']
if 'get_prob' in data_json:
disease_dict['get_prob'] = data_json['get_prob']
if 'easy_get' in data_json:
disease_dict['easy_get'] = data_json['easy_get']
#如果对于一个疾病对应了门诊部，还要记录部门之间的关系
if 'cure_department' in data_json:
cure_department = data_json['cure_department']
if len(cure_department) == 1:
rels_category.append([disease, cure_department[0]])
if len(cure_department) == 2:
big = cure_department[0]
small = cure_department[1]
rels_department.append([small, big])
rels_category.append([disease, small])
disease_dict['cure_department'] = cure_department
departments += cure_department
if 'cure_way' in data_json:
disease_dict['cure_way'] = data_json['cure_way']
if 'cure_lasttime' in data_json:
disease_dict['cure_lasttime'] = data_json['cure_lasttime']
if 'cured_prob' in data_json:
disease_dict['cured_prob'] = data_json['cured_prob']
if 'common_drug' in data_json:
common_drug = data_json['common_drug']
for drug in common_drug:
rels_commonddrug.append([disease, drug])
drugs += common_drug
if 'recommand_drug' in data_json:
recommand_drug = data_json['recommand_drug']
drugs += recommand_drug
for drug in recommand_drug:
rels_recommanddrug.append([disease, drug])
if 'not_eat' in data_json:
not_eat = data_json['not_eat']
for _not in not_eat:
rels_noteat.append([disease, _not])
foods += not_eat
do_eat = data_json['do_eat']
for _do in do_eat:
rels_doeat.append([disease, _do])
foods += do_eat
recommand_eat = data_json['recommand_eat']
for _recommand in recommand_eat:
rels_recommandeat.append([disease, _recommand])
foods += recommand_eat
if 'check' in data_json:
check = data_json['check']
for _check in check:
rels_check.append([disease, _check])
checks += check
if 'drug_detail' in data_json:
drug_detail = data_json['drug_detail']
producer = [i.split('(')[0] for i in drug_detail]
rels_drug_producer += [[i.split('(')[0], i.split('(')[-1].replace(')', '')] for i in drug_detail]
producers += producer
disease_infos.append(disease_dict)
return set(drugs), set(foods), set(checks), set(departments), set(producers), set(symptoms), set(diseases), disease_infos,\
rels_check, rels_recommandeat, rels_noteat, rels_doeat, rels_department, rels_commonddrug, rels_drug_producer, rels_recommanddrug,\
rels_symptom, rels_acompany, rels_category
'''建立节点'''
def create_node(self, label, nodes):
count = 0
for node_name in nodes:
node = Node(label, name=node_name)
self.g.create(node)
count += 1
#print(count, len(nodes))
return
'''创建知识图谱中心疾病的节点'''
def create_diseases_nodes(self, disease_infos):
count = 0
#print(disease_infos)
for disease_dict in disease_infos:
node = Node("Disease", name=disease_dict['name'], desc=disease_dict['desc'],
prevent=disease_dict['prevent'] ,cause=disease_dict['cause'],
easy_get=disease_dict['easy_get'],cure_lasttime=disease_dict['cure_lasttime'],
cure_department=disease_dict['cure_department']
,cure_way=disease_dict['cure_way'] , cured_prob=disease_dict['cured_prob'])
self.g.create(node)
count += 1
#print(count)
return
'''创建知识图谱实体节点类型schema'''
def create_graphnodes(self):
#调用read_nodes()：因为该函数的返回值很多，所以需要一堆变量接收
Drugs, Foods, Checks, Departments, Producers, Symptoms, Diseases, disease_infos,rels_check, rels_recommandeat, rels_noteat, rels_doeat, rels_department, rels_commonddrug, rels_drug_producer, rels_recommanddrug,rels_symptom, rels_acompany, rels_category = self.read_nodes()
#创建疾病类结点
self.create_diseases_nodes(disease_infos)
#创建其他所有结点
self.create_node('Drug', Drugs)
self.create_node('Food', Foods)
self.create_node('Check', Checks)
self.create_node('Department', Departments)
self.create_node('Producer', Producers)
self.create_node('Symptom', Symptoms)
return
'''创建实体关系边'''
def create_graphrels(self):
#调用read_nodes()：因为该函数的返回值很多，所以需要一堆变量接收
Drugs, Foods, Checks, Departments, Producers, Symptoms, Diseases, disease_infos, rels_check, rels_recommandeat, rels_noteat, rels_doeat, rels_department, rels_commonddrug, rels_drug_producer, rels_recommanddrug,rels_symptom, rels_acompany, rels_category = self.read_nodes()
self.create_relationship('Disease', 'Food', rels_recommandeat, 'recommand_eat', '推荐食谱')
self.create_relationship('Disease', 'Food', rels_noteat, 'no_eat', '忌吃')
self.create_relationship('Disease', 'Food', rels_doeat, 'do_eat', '宜吃')
self.create_relationship('Department', 'Department', rels_department, 'belongs_to', '属于')
self.create_relationship('Disease', 'Drug', rels_commonddrug, 'common_drug', '常用药品')
self.create_relationship('Producer', 'Drug', rels_drug_producer, 'drugs_of', '生产药品')
self.create_relationship('Disease', 'Drug', rels_recommanddrug, 'recommand_drug', '好评药品')
self.create_relationship('Disease', 'Check', rels_check, 'need_check', '诊断检查')
self.create_relationship('Disease', 'Symptom', rels_symptom, 'has_symptom', '症状')
self.create_relationship('Disease', 'Disease', rels_acompany, 'acompany_with', '并发症')
self.create_relationship('Disease', 'Department', rels_category, 'belongs_to', '所属科室')
'''创建实体关联边'''
def create_relationship(self, start_node, end_node, edges, rel_type, rel_name):
count = 0
# 去重处理
set_edges = []
for edge in edges:
#edge是关系，包含两实体，以“###为分隔符，将edge的每一字符分隔开”
set_edges.append('###'.join(edge))
#print(set_edges)
all = len(set(set_edges))
for edge in set(set_edges):
edge = edge.split('###')
p = edge[0]
q = edge[1]
query = "match(p:%s),(q:%s) where p.name='%s'and q.name='%s' create (p)-[rel:%s{name:'%s'}]->(q)" % (
start_node, end_node, p, q, rel_type, rel_name)
try:
self.g.run(query)
count += 1
# print(rel_type, count, all)
except Exception as e:
print(e)
return
if __name__ == '__main__':
handler = MedicalGraph()
handler.create_graphnodes()
handler.create_graphrels()
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
- 72
- 73
- 74
- 75
- 76
- 77
- 78
- 79
- 80
- 81
- 82
- 83
- 84
- 85
- 86
- 87
- 88
- 89
- 90
- 91
- 92
- 93
- 94
- 95
- 96
- 97
- 98
- 99
- 100
- 101
- 102
- 103
- 104
- 105
- 106
- 107
- 108
- 109
- 110
- 111
- 112
- 113
- 114
- 115
- 116
- 117
- 118
- 119
- 120
- 121
- 122
- 123
- 124
- 125
- 126
- 127
- 128
- 129
- 130
- 131
- 132
- 133
- 134
- 135
- 136
- 137
- 138
- 139
- 140
- 141
- 142
- 143
- 144
- 145
- 146
- 147
- 148
- 149
- 150
- 151
- 152
- 153
- 154
- 155
- 156
- 157
- 158
- 159
- 160
- 161
- 162
- 163
- 164
- 165
- 166
- 167
- 168
- 169
- 170
- 171
- 172
- 173
- 174
- 175
- 176
- 177
- 178
- 179
- 180
- 181
- 182
- 183
- 184
- 185
- 186
- 187
- 188
- 189
- 190
- 191
- 192
- 193
- 194
- 195
- 196
- 197
- 198
- 199
- 200
- 201
- 202
- 203
- 204
- 205
- 206
- 207
- 208
- 209
- 210
- 211
- 212
- 213
- 214
- 215
- 216
- 217
- 218
- 219
- 220
- 221
- 222
- 223
- 224
- 225
- 226
- 227
- 228
- 229
- 230
- 231
- 232
- 233
- 234
- 235
- 236
- 237
- 238
- 239
- 240
- 241
- 242
- 243
- 244
- 245
- 246
- 247
医药问答系统完整项目下载地址：
https://download.csdn.net/download/floracuu/15927225?spm=1001.2014.3001.5501
声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/AllinToyou/article/detail/363512
推荐阅读
相关标签
