大数据开发-订单表数据清洗案例

回顾上节简单介绍了什么是数据仓库以及数仓数据同步工具，下面简单弄个demo

数据开发

ODS层

## 建表
create database if not exists ods_mall;
create external table if not exists ods_mall.ods_user_order(
order_id bigint,
user_id bigint,
order_money bigint,
order_type string,
order_status string,
order_time string,
order_update_time string
) partitioned by(dt string)
row format delimited
fields terminated by '\t'
location 'hdfs://hadoop01:9000/data/ods/user_order';

# 建分区
alter table dwd_mall.dwd_user_order add if not exists partition(dt='20240318') location 'hdfs://hadoop01:9000/data/ods/user_order/20240318'

# 数据导入HDFS sqoop_collect_data.sh
#!/bin/bash
if [ $# != 2 ]
then
echo "参数异常:<sql><hdfs_path>"
exit 100
fi
## 数据sql 
sql=$1
# 路径
hdfs_path=$2
sqoop import \
--connect jdbc:mysql://ip:port/test?serverTimezone=UTC \
--username root \
--password 123456 \
--target-dir "${hdfs_path}" \
--delete-target-dir \
--num-mappers 1 \
--fields-terminated-by '\t' 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36