数仓dwd层事件日志dwd_base_event_log解析生成若干表

需要创建的表

dwd_display_log 商品点击表
dwd_newsdetail_log 商品详情页表
dwd_loading_log 商品列表页表
dwd_ad_log 广告表
dwd_notification_log 消息通知表
dwd_active_background_log 用户后台活跃表
dwd_comment_log 评论表
dwd_favorites_log 收藏表
dwd_praise_log 点赞表
dwd_error_log 错误日志表

商品详情页表

#创建表dwd_newsdetail_log
drop table if exists dwd_newsdetail_log; 
create external table dwd_newsdetail_log( 
`mid_id` string, 
`user_id` string, 
`version_code` string, 
`version_name` string, 
`lang` string, 
`source` string, 
`os` string, 
`area` string, 
`model` string, 
`brand` string, 
`sdk_version` string, 
`gmail` string, 
`height_width` string, 
`app_time` string, 
`network` string, 
`lng` string, 
`lat` string, 
`entry` string, 
`action` string, 
`goodsid` string, 
`showtype` string, 
`news_staytime` string,
`loading_time` string, 
`type1` string, 
`category` string, 
`server_time` string
) 
partitioned by (`dt` string) stored as parquet 
location '/warehouse/gmall/dwd/dwd_newsdetail_log/'; 

#解析并导入数据
insert overwrite table dwd_newsdetail_log
partition(`dt`='2020-02-01')
select 
mid_id, 
user_id, 
version_code, 
version_name, 
lang, 
source, 
os, 
area, 
model, 
brand, 
sdk_version, 
gmail, 
height_width, 
app_time, 
network, 
lng, 
lat, 
get_json_object(event_json,'$.kv.entry') entry, get_json_object(event_json,'$.kv.action') action, get_json_object(event_json,'$.kv.goodsid') goodsid, get_json_object(event_json,'$.kv.showtype') showtype, get_json_object(event_json,'$.kv.news_staytime') news_staytime, get_json_object(event_json,'$.kv.loading_time') loading_time, get_json_object(event_json,'$.kv.type1') type1, get_json_object(event_json,'$.kv.category') category, 
server_time 
from dwd_base_event_log
where `dt`='2020-02-01' and event_name='newsdetail';

#查询导入结果
select * from dwd_newsdetail_log limit 2;
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

商品列表页表

drop table if exists dwd_loading_log;
create external table dwd_loading_log(
`mid_id` string, 
`user_id` string, 
`version_code` string, 
`version_name` string, 
`lang` string, 
`source` string, 
`os` string, 
`area` string,
`model` string, 
`brand` string, 
`sdk_version` string, 
`gmail` string, 
`height_width` string, 
`app_time` string, 
`network` string, 
`lng` string, 
`lat` string, 
`action` string, 
`loading_time` string, 
`loading_way` string, 
`extend1` string, 
`extend2` string, 
`type` string, 
`type1` string, 
`server_time` string
) 
partitioned by (`dt` string)
stored as parquet
location '/warehouse/gmall/dwd/dwd_loading_log';

#导入数据
insert overwrite table dwd_loading_log
partition(`dt`='2020-02-01')
select
mid_id, 
user_id, 
version_code, 
version_name, 
lang, 
source, 
os, 
area, 
model, 
brand, 
sdk_version, 
gmail, 
height_width, 
app_time, 
network, 
lng, 
lat, 
get_json_object(event_json,'$.kv.action') action, get_json_object(event_json,'$.kv.loading_time') loading_time, get_json_object(event_json,'$.kv.loading_way') loading_way, get_json_object(event_json,'$.kv.extend1') extend1, get_json_object(event_json,'$.kv.extend2') extend2, get_json_object(event_json,'$.kv.type') type, get_json_object(event_json,'$.kv.type1') type1, 
server_time 
from dwd_base_event_log
where `dt`='2020-02-01' and event_name='loading';
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

广告表

drop table if exists  dwd_ad_log;
create external table  dwd_ad_log(
`mid_id` string, 
`user_id` string, 
`version_code` string, 
`version_name` string, 
`lang` string, 
`source` string, 
`os` string, 
`area` string, 
`model` string, 
`brand` string, 
`sdk_version` string, 
`gmail` string, 
`height_width` string, 
`app_time` string, 
`network` string, 
`lng` string, 
`lat` string, 
`entry` string, 
`action` string, 
`contentType` string, 
`displayMills` string, 
`itemId` string, 
`activityId` string, 
`server_time` string
) 
partitioned by (`dt` string)
stored as parquet
location '/warehouse/gmall/dwd/dwd_ad_log';

#导入数据
insert overwrite table dwd_ad_log
partition (`dt`='2020-02-01')
select
mid_id, 
user_id, 
version_code, 
version_name, 
lang, 
source, 
os, 
area, 
model, 
brand, 
sdk_version, 
gmail, 
height_width, 
app_time, 
network, 
lng, 
lat, 
get_json_object(event_json,'$.kv.entry') entry, get_json_object(event_json,'$.kv.action') action, get_json_object(event_json,'$.kv.contentType') contentType, get_json_object(event_json,'$.kv.displayMills') displayMills, get_json_object(event_json,'$.kv.itemId') itemId, get_json_object(event_json,'$.kv.activityId') activityId, 
server_time
from dwd_base_event_log
where `dt`='2020-02-01' and event_name='ad';
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

消息通知表

#创建表dwd_notification_log
drop table if exists dwd_notification_log; 
create external table dwd_notification_log( 
`mid_id` string, 
`user_id` string, 
`version_code` string, 
`version_name` string, 
`lang` string, 
`source` string, 
`os` string, 
`area` string, 
`model` string, 
`brand` string, 
`sdk_version` string, 
`gmail` string, 
`height_width` string, `
app_time` string, 
`network` string, 
`lng` string, 
`lat` string, 
`action` string, 
`noti_type` string, 
`ap_time` string, 
`content` string, 
`server_time` string 
) 
partitioned by (dt string) stored as parquet 
location '/warehouse/gmall/dwd/dwd_notification_log/';

#导入数据
insert overwrite table dwd_notification_log 
partition(dt='2020-02-01') 
select mid_id, 
user_id, 
version_code, 
version_name, 
lang, 
source, 
os, 
area, 
model, 
brand, 
sdk_version, 
gmail, 
height_width, 
app_time, 
network,
lng, lat, 
get_json_object(event_json,'$.kv.action') action, get_json_object(event_json,'$.kv.noti_type') noti_type, get_json_object(event_json,'$.kv.ap_time') ap_time, get_json_object(event_json,'$.kv.content') content, 
server_time 
from dwd_base_event_log 
where dt='2020-02-01' and event_name='notification'; 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

用户后台活跃表

#创建表
drop table if exists dwd_active_background_log; 
create external table dwd_active_background_log( 
`mid_id` string, 
`user_id` string, 
`version_code` string, 
`version_name` string, 
`lang` string, 
`source` string, 
`os` string, 
`area` string, 
`model` string, 
`brand` string, 
`sdk_version` string, 
`gmail` string, 
`height_width` string, 
`app_time` string, 
`network` string, 
`lng` string, 
`lat` string, 
`active_source` string, 
`server_time` string 
) 
partitioned by (`dt` string) 
stored as parquet 
location '/warehouse/gmall/dwd/dwd_background_log/';
 
#导入数据
insert overwrite table dwd_active_background_log 
partition (dt='2020-02-01') 
select 
mid_id, 
user_id, 
version_code, 
version_name, 
lang, 
source, 
os, 
area, model,
brand, 
sdk_version, 
gmail, 
height_width, 
app_time, 
network, 
lng, 
lat, 
get_json_object(event_json,'$.kv.active_source') active_source, server_time 
from dwd_base_event_log 
where dt='2020-02-01' and event_name='active_background'; 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

评论表

#创建表
drop table if exists dwd_comment_log; 
create external table dwd_comment_log( 
`mid_id` string, 
`user_id` string, 
`version_code` string, 
`version_name` string, 
`lang` string, 
`source` string, 
`os` string, 
`area` string, 
`model` string, 
`brand` string, 
`sdk_version` string, 
`gmail` string, 
`height_width` string, 
`app_time` string, 
`network` string, 
`lng` string, 
`lat` string, 
`comment_id` int, 
`userid` int, 
`p_comment_id` int, 
`content` string, 
`addtime` string, 
`other_id` int, 
`praise_count` int, 
`reply_count` int, 
`server_time` string 
) 
partitioned by (`dt` string) 
stored as parquet 
location '/warehouse/gmall/dwd/dwd_comment_log/'; 

#导入数据
insert overwrite table dwd_comment_log 
partition (dt='2020-02-01') 
select
mid_id, 
user_id, 
version_code, 
version_name, 
lang, 
source, 
os, 
area, 
model, 
brand, 
sdk_version, 
gmail, 
height_width, 
app_time, 
network, 
lng, 
lat, 
get_json_object(event_json,'$.kv.comment_id') comment_id, get_json_object(event_json,'$.kv.userid') userid, get_json_object(event_json,'$.kv.p_comment_id') p_comment_id, get_json_object(event_json,'$.kv.content') content, get_json_object(event_json,'$.kv.addtime') addtime, get_json_object(event_json,'$.kv.other_id') other_id, get_json_object(event_json,'$.kv.praise_count') praise_count, get_json_object(event_json,'$.kv.reply_count') reply_count, 
server_time 
from dwd_base_event_log 
where dt='2020-02-01' and event_name='comment'; 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

收藏表

#创建表
drop table if exists dwd_favorites_log; 
CREATE EXTERNAL TABLE dwd_favorites_log( 
`mid_id` string, 
`user_id` string, 
`version_code` string, 
`version_name` string, 
`lang` string, 
`source` string, 
`os` string, 
`area` string, 
`model` string, 
`brand` string, 
`sdk_version` string, 
`gmail` string, 
`height_width` string, 
`app_time` string, 
`network` string, 
`lng` string, 
`lat` string, 
`id` int, 
`course_id` int, 
`userid` int, 
`add_time` string,
`server_time` string
) 
partitioned by (`dt` string) 
stored as parquet 
location '/warehouse/gmall/dwd/dwd_favorites_log/';
 
#导入数据
insert overwrite table dwd_favorites_log 
partition (`dt`='2020-02-01') 
select 
mid_id, 
user_id, 
version_code, 
version_name, 
lang, 
source, 
os, 
area,
model, 
brand, 
sdk_version, 
gmail, 
height_width, 
app_time, 
network, 
lng, 
lat, 
get_json_object(event_json,'$.kv.id') id, get_json_object(event_json,'$.kv.course_id') course_id, get_json_object(event_json,'$.kv.userid') userid, get_json_object(event_json,'$.kv.add_time') add_time, 
server_time 
from dwd_base_event_log 
where dt='2020-02-01' and event_name='favorites'; 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

点赞表

#创建表
drop table if exists dwd_praise_log; 
CREATE EXTERNAL TABLE dwd_praise_log( 
`mid_id` string, 
`user_id` string, 
`version_code` string, 
`version_name` string, 
`lang` string, 
`source` string, 
`os` string, 
`area` string, 
`model` string, 
`brand` string, 
`sdk_version` string, 
`gmail` string, 
`height_width` string,
`app_time` string, 
`network` string, 
`lng` string, 
`lat` string, 
`id` string, 
`userid` string, 
`target_id` string, 
`type` string, 
`add_time` string, 
`server_time` string
) 
partitioned by(`dt` string) 
stored as parquet 
location '/warehouse/gmall/dwd/dwd_praise_log/'; 

#导入数据
insert overwrite table dwd_praise_log 
PARTITION (dt='2020-02-01') 
select 
mid_id, 
user_id, 
version_code, 
version_name, 
lang, 
source, 
os, 
area, 
model, 
brand, 
sdk_version, 
gmail, 
height_width, 
app_time, 
network, 
lng, 
lat, 
get_json_object(event_json,'$.kv.id') id, get_json_object(event_json,'$.kv.userid') userid, get_json_object(event_json,'$.kv.target_id') target_id, get_json_object(event_json,'$.kv.type') type, get_json_object(event_json,'$.kv.add_time') add_time, 
server_time 
from dwd_base_event_log 
where dt='2020-02-01' and event_name='praise'; 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

错误日志表

#创建表
drop table if exists dwd_error_log; 
CREATE EXTERNAL TABLE dwd_error_log( 
`mid_id` string, 
`user_id` string, 
`version_code` string, 
`version_name` string,
`lang` string, 
`source` string, 
`os` string, 
`area` string, 
`model` string, 
`brand` string, 
`sdk_version` string, 
`gmail` string, 
`height_width` string, 
`app_time` string, 
`network` string, 
`lng` string, 
`lat` string, 
`errorBrief` string, 
`errorDetail` string, 
`server_time` string
) 
partitioned by (`dt` string) 
stored as parquet 
location '/warehouse/gmall/dwd/dwd_error_log/';
 
#导入数据
insert overwrite table dwd_error_log 
partition(`dt`='2020-02-01') 
select 
mid_id, 
user_id, 
version_code, 
version_name, 
lang, 
source, 
os, 
area, 
model, 
brand, 
sdk_version, 
gmail, 
height_width, 
app_time, 
network, 
lng, 
lat, 
get_json_object(event_json,'$.kv.errorBrief') errorBrief, get_json_object(event_json,'$.kv.errorDetail') errorDetail, 
server_time 
from dwd_base_event_log 
where dt='2020-02-01' and event_name='error'; 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53