如何使用 datax 将 mysql 中的数据拉取到 hive ？_datax同步数据到hive依赖包

需求

使用datax将mysql中的数据拉取到hive的ods层

步骤

首先在mysql中确定好需要拉取的表user_extend，
然后对应在hive中创建好空表，等待拉取

这里对应创建的hive表格如下

CREATE TABLE snbap_ods.ods_user_extend (
  user_id bigint   ,
  user_gender bigint   ,
  is_pregnant_woman tinyint  ,
  is_have_children tinyint  ,
  is_have_car tinyint  ,
  phone_brand string  ,
  phone_brand_level string  ,
  phone_cnt int   ,
  change_phone_cnt int   ,
  is_maja tinyint  ,
  majia_account_cnt int   ,
  loyal_model string  ,
  shopping_type_model string  ,
  weight int   ,
  height int   
)  
stored as textfile
location '/datax/mysql/snbap_ods/ods/ods_user_extend';

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

在datax的bin目录下创建json脚本 vi datax1.json

{
   
    "job": {
   
        "setting": {
   
             "speed": {
   
                "byte":1048576,
                "channel":"4"
            }
        },
        "content": [
            {
   
                "reader": {
   
                    "name": "mysqlreader",
                    "parameter": {
   
                        "username": "root",
                        "password": "root",
                        "connection": [
                            {
   
                                "querySql": [
                                    "select * from user_extend"
                                ],
                                "jdbcUrl": [
                                    "jdbc:mysql://localhost:3306/snbap_ods"
                                ]
                            }
                        ]
                    }
                },
                "writer": {
   
                    "name": "hdfswriter",
                    "parameter": {
   
                        "defaultFS":"hdfs://192.168.1.54:9000",
	        "fileType":"text",
	        "path":"/datax/mysql/snbap_ods/ods/ods_user_extend/",
	        "fileName":"user_extend",
                        "column": [
                        {
   "name":"user_id","type":"bigint"}
                        {
   "name":"user_gender","type":"bigint"}
                        {
   "name":"is_pregnant_woman","type":"tinyint"}
                        {
   "name":"is_have_children","type":"tinyint"}
                        {
   "name":"is_have_car","type":"tinyint"}
                        {
   "name":"phone_brand","type":"string"}
                        {
   "name":"phone_brand_level","type":"string"}
                        {
   "name":"phone_cnt","type":"int"}
                        {
   "name":"change_phone_cnt","type":"int"}
                        {
   "name":"is_maja","type":"tinyint"}
                        {
   "name":"majia_account_cnt","type":"int"}
                        {
   "name":"loyal_model","type":"string"}
                        {
   "name":"shopping_type_model","type":"string"}
                        {
   "name":"weight","type":"int"}
                        {
   "name":"height","type":"int"}
                        ],
                    "writeMode":"append",
         "fieldDelimiter":"\u0001",
	    "compress":"gzip"
                    }
                }
            }
        ]
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

在datax的bin目录下运行以上json脚本

python /opt/install/datax/bin/datax.py /opt/install/datax