Tez调优参数总结

hive.tez.auto.reducer.parallelism=false
hive.tez.bucket.pruning=false
hive.tez.bucket.pruning.compat=true
hive.tez.container.max.java.heap.fraction=0.8
hive.tez.container.size=768
hive.tez.cpu.vcores=-1
hive.tez.dynamic.partition.pruning=true
hive.tez.dynamic.partition.pruning.max.data.size=104857600
hive.tez.dynamic.partition.pruning.max.event.size=1048576
hive.tez.enable.memory.manager=true
hive.tez.exec.inplace.progress=true
hive.tez.exec.print.summary=false

如果map阶段划分过多，可以设置为合并小文件格式化org.apache.hadoop.hive.ql.io.CombineHiveInputFormat
hive.tez.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat
hive.tez.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat

hive.tez.input.generate.consistent.splits=true
hive.tez.java.opts=-Xmx1555m
hive.tez.log.level=INFO
hive.tez.max.partition.factor=2.0
hive.tez.min.partition.factor=0.25
hive.tez.smb.number.waves=0.5
hive.tez.task.scale.memory.reserve-fraction.min=0.3
hive.tez.task.scale.memory.reserve.fraction=-1.0
hive.tez.task.scale.memory.reserve.fraction.max=0.5

maxReduces = min(hive.exec.reducers.max [1099], max((ReducerStage estimate/hive.exec.reducers.bytes.per.reducer),1)*hive.tez.max.partition.factor)

minReduces = min(hive.exec.reducers.max [1099], max((ReducerStage estimate/hive.exec.reducers.bytes.per.reducer),1)*hive.tez.min.partition.factor)

set hive.tez.container.size=4096;
set hive.tez.java.opts=-Xmx1024m;
set tez.am.java.opts=-Xmx4096m;
set tez.am.resource.memory.mb=2048m;

set tez.am.task.max.failed.attempts=10;
set tez.am.max.app.attemps=5;

set tez.am.resource.memory.mb=768;
set tez.am.java.opts=-Xmx615m;
set hive.tez.container.size=768;
set hive.tez.java.opts=-Xmx615m;

如果一直报java heap space内存溢出问题，则一直增加这个java内存值即可，其他参数不用作修改，以是该系统配置的最优
set hive.tez.java.opts=-Xmx6144m;