datax数据迁移mysql到mongo_datax mongodb

不写版本的文章都是耍流氓
MongoDB4.2.8
mysql 5.7
最近有个需求是把mysql的多余数据迁移到mongo
docker 安装MongoDB4.2.8
1下载镜像
docker pull mongo:4.2.8
2启动
docker run --name mongo-master -p 27017:27017 -v /etc/localtime:/etc/localtime:ro -v /home/mongo/db:/data/db -d mongo:4.2.8 --auth
设置账号密码
docker exec -it mongo /bin/bash
mongo
use admin
db.createUser({user: “root”, pwd: “123456”, roles: [{role: “root”, db: “admin”}]})
db.auth(‘root’,‘123456’)

use stpnew
db.createCollection(“att_record”)
db.createUser({user:“att”,pwd:“123456”,roles:[{role:‘dbOwner’,db:‘stpnew’}]})
datax下载地址
https://github.com/alibaba/DataX
自定义sql ，根据条件转移相关数据 只要配置json就行

{
	"job": {
		"setting": {
			"speed": {
				"channel": 1
			}
		},
		"content": [{
			"reader": {
				"name": "mysqlreader",
				"parameter": {
					"username": "root",
					"password": "12345",
					"connection": [{
					 "querySql": [
                                    "SELECT id,user_id, IFNULL(number,'') AS number ,username,parkinglot_id,swipe_time,swipe_type,CONCAT(lng) AS lng ,CONCAT(lat) AS lat,ip,device_no,ver,createtime,IFNULL(address,'') AS address FROM att_record  WHERE address IS  NULL;"
                                ],
						
						"jdbcUrl": [
							"jdbc:mysql://127.0.0.1:3306/stpnew"
						]
					}]
				}
			},
			"writer": {
				"name": "mongodbwriter",
				"parameter": {
					"address": [
						"localhost:27017"
					],
					"userName": "att",
					"userPassword": "123456",
					"dbName": "stpnew",
					"collectionName": "att_record",
					"column": [{
							"name": "id",
							"type": "int"
						},
						{
							"name": "user_id",
							"type": "int"
						},
						{
							"name": "number",
							"type": "string"
						},
						{
							"name": "username",
							"type": "string"
						},
						{
							"name": "parkinglot_id",
							"type": "int"
						},
						{
							"name": "swipe_time",
							"type": "date"
						},
						{
							"name": "swipe_type",
							"type": "int"
						},
						{
							"name": "lng",
							"type": "String"
						},
						{
							"name": "lat",
							"type": "String"
						},
						{
							"name": "ip",
							"type": "string"
						}
						,
						{
							"name": "device_no",
							"type": "string"
						}
						,
						{
							"name": "ver",
							"type": "string"
						}
						
						,
						{
							"name": "createtime",
							"type": "date"
						}
						,
						
						{
							"name": "address",
							"type": "string"
						}
					],
					"writeMode": {
						"isReplace": "true",
						"replaceKey": "id"
					}
				}
			}
		}]
	}
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107

执行

python2 bin/datax.py job/job.json
datax的程序使用Python2写的如果用3的话需要修改脚本
结果