赞
踩
例如 我需要造的数据模型如下
其中 msg_creation_time ,qualifiedName ,name,guid 是变化的
{ "version": { "version": "1.0.0", "versionParts": [ 1 ] }, "msg_compression_kind": "1", "msg_split_idx": 1, "msg_source_ip": "10.82.13.74", "msg_created_by": "DELL", "msg_creation_time": 1711525684104, "message": { "type": "ENTITY_CREATE_V2", "user": "DELL", "entities": { "entities": [ { "typeName": "ceshifuzadu2", "attributes": { "qualifiedName": "1711525684104", "tenantId": "tenantId12", "name": "百万消息测试复杂度2024-03-27 07:48:04.104966+00", "userId": "userId12", "version": 1 }, "guid": "-1711525684104", "isIncomplete": false, "provenanceType": 0, "version": 0, "proxy": false } ] } } }
create table test_b
(
version json,
msg_compression_kind varchar,
msg_split_idx integer,
msg_source_ip varchar,
msg_created_by varchar,
msg_creation_time bigint,
message json
);
DO $$ DECLARE k integer := 1; DECLARE message json :='{}'; begin WHILE k <= 1000000 loop WITH dynamic_json AS ( SELECT json_build_object( 'type', 'ENTITY_CREATE_V2', 'user', 'DELL', 'entities', json_build_object( 'entities', json_build_array( json_build_object( 'typeName', 'ceshifuzadu2', 'attributes', json_build_object( 'qualifiedName', ''||FLOOR(EXTRACT(EPOCH FROM NOW()) * 1000)+k, 'tenantId', 'tenantId12', 'name', '百万消息测试复杂度' || FLOOR(EXTRACT(EPOCH FROM NOW()) * 1000)+k, 'userId', 'userId12', 'version', 1 ), 'guid', '-' || FLOOR(EXTRACT(EPOCH FROM NOW()) * 1000)+k, 'isIncomplete', false, 'provenanceType', 0, 'version', 0, 'proxy', false ) ) ) ) AS messageb ) SELECT messageb INTO message FROM dynamic_json; INSERT INTO public.test_b ("version", "msg_compression_kind", "msg_split_idx", "msg_source_ip", "msg_created_by", "msg_creation_time", "message") VALUES('{"version":"1.0.0","versionParts":[1]}', 1, 1, '10.82.13.74','DELL',FLOOR(EXTRACT(EPOCH FROM NOW()) * 1000), message); k = k + 1; end loop; END $$;
## psql 登录
psql -U postgre -w
## 切换数据库
\c database;
## 执行导出数据的命令
COPY (select row_to_json(t) from test_b t) to '/tmp/testb.json';
/opt/kafka_2.13-2.8.1/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic ATLAS_HOOK < /tmp/testb.json
Copyright © 2003-2013 www.wpsshop.cn 版权所有,并保留所有权利。