redis_exporter部署记录_redis-exporter

redis_exporter官网链接
https://github.com/oliver006/redis_exporter

1 docker部署redis_exporter

docker run  -d --name redis_exporter_dev  -p 9121:9121 oliver006/redis_exporter:v1.45.0 --redis.addr redis://10.10.51.94 --redis.password "admin123456"
docker run  -d --name redis_exporter_test  -p 9122:9121 oliver006/redis_exporter:v1.45.0 --redis.addr redis://10.10.51.134 --redis.password "admin123456"
docker run  -d --name redis_exporter_uat  -p 9123:9121 oliver006/redis_exporter:v1.45.0 --redis.addr redis://10.10.51.134:6380 --redis.password "admin123456"
docker run  -d --name redis_exporter_prod  -p 9124:9121 oliver006/redis_exporter:v1.45.0 --redis.addr redis://10.10.51.136--redis.password "admin123456"

镜像存在docker hub上，如果无法正常连接，可以通过quay.io docker repo下载镜像
https://quay.io/repository/oliver006/redis_exporter?tab=tags&tag=latest
1
2
3
4
5
6
7

#使用docker-compose
  redis_exporter_dev:
    image: oliver006/redis_exporter:v1.45.0
    restart: always
    command:
      - --redis.addr=redis://10.10.51.94
      - --redis.password=admin123456
    networks:
      - prometheus-network
  redis_exporter_test:
    image: oliver006/redis_exporter:v1.45.0
    restart: always
    command:
      - --redis.addr=redis://10.10.51.134
      - --redis.password=admin123456
    networks:
      - prometheus-network
  redis_exporter_uat:
    image: oliver006/redis_exporter:v1.45.0
    restart: always
    command:
      - --redis.addr=redis://10.10.51.134:6380
      - --redis.password=admin123456
    networks:
      - prometheus-network
  redis_exporter_prod:
    image: oliver006/redis_exporter:v1.45.0
    restart: always
    command:
      - --redis.addr=redis://10.10.51.136
      - --redis.password=admin123456
    networks:
      - prometheus-network
networks:
  prometheus-network:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

exporter配置参数如下

2 exporter暴漏的指标

• INFO命令输出的信息大部分会暴漏
• 每个数据库的所有的key相关指标、过期key、key的平均TTL

3 Prometheus监控redis

docker-compose配置Prometheus

version: '3.2'
services:
  prometheus:
    image: prom/prometheus:v2.41.0
    restart: "always"
    user: "0:0"
    ports:
      - 9090:9090
    volumes:
      - "./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml"
      - "./prometheus/k3s.token:/etc/prometheus/k3s.token"
      - "./rules:/etc/prometheus/rules"
      - "./prometheus/data:/prometheus"
      - /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro
    command:
      - '--web.read-timeout=5m' #请求连接最大等待时间，防止太多空连接占用资源
      - '--web.max-connections=512' #最大链接数
      - '--config.file=/etc/prometheus/prometheus.yml'       #   设置yml路径  跟上面挂载对应
      - '--storage.tsdb.path=/prometheus'                     #设置数据路径   跟上面挂载对应
      - '--storage.tsdb.retention=7d'
      - '--query.timeout=2m'
      - '--query.max-concurrency=20' #用户查询优化
      - --web.enable-lifecycle  #开启webapi
    networks:
      - prometheus-network
  #告警模块
  alertmanager:
    image: prom/alertmanager:v0.25.0
    restart: "always"
    user: '0:0'
    ports:
      - 9093:9093
      - 8001:8001
    volumes:
      - "./alert/alertmanager.yml:/etc/alertmanager/alertmanager.yml"
      - /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro
    command:
      - '--cluster.peer=alertmanager:8001'
      - '--cluster.listen-address=0.0.0.0:8001'
      - '--config.file=/etc/alertmanager/alertmanager.yml'
    depends_on:
      - prometheus
    networks:
      - prometheus-network
  #web界面
  grafana:
    image: grafana/grafana:9.3.2
    restart: "always"
    user: '0:0'
    ports:
      - 3000:3000
    volumes:
      - "./grafana/grafana.ini:/etc/grafana/grafana.ini"              #配置文件自行拷贝出来
      - "./grafana/grafana-storage:/var/lib/grafana"
      - /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro
    depends_on:
      - prometheus
    networks:
      - prometheus-network
  # 监控MySQL
  mysqld-exporter1:
    image: prom/mysqld-exporter:v0.15.0
    restart: always
    user: "0:0"
    volumes:
      - ./mysql/mysql-dev.cnf:/my.cnf
      - /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro
    command: --config.my-cnf=/my.cnf
    networks:
      - prometheus-network
  #监控mongo
  mongod-exported-uat:
    image: bitnami/mongodb-exporter
    restart: always
    command:
      - "--mongodb.uri=mongodb://prometheus:prometheus@10.10.51.134:27017"
      - "--web.listen-address=:9104"
      - "--collect-all"
      - --compatible-mode
    volumes:
      - /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro
    networks:
      - prometheus-network
  #监控redis
  redis_exporter_dev:
    image: oliver006/redis_exporter:v1.45.0
    restart: always
    command:
      - --redis.addr=redis://10.10.51.94
      - --redis.password=123456
    networks:
      - prometheus-network
  redis_exporter_test:
    image: oliver006/redis_exporter:v1.45.0
    restart: always
    command:
      - --redis.addr=redis://10.10.51.134
      - --redis.password=123456
    networks:
      - prometheus-network
  #企业微信告警
  wechat:
    image: guyongquan/webhook-adapter
    restart: always
    command: --adapter=/app/prometheusalert/wx.js=/wx=https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=b86ca170-85bf-4f2f-994e-fae73c856947
    networks:
      - prometheus-network
networks:
  prometheus-network:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

prometheus.yml

global:
  scrape_interval:     10s    # 多久 收集 一次数据
  evaluation_interval: 15s    # 多久 评估 一次规则
  scrape_timeout:      10s    # 每次 收集数据的 超时时间

# 收集数据 配置 列表
scrape_configs:

  - job_name: 'redis-dev'
    static_configs:
      - targets: [ 'redis_exporter_dev:9121' ]
        labels:
          env: dev
          instance: redis-dev
  - job_name: 'redis-test'
    static_configs:
      - targets: [ 'redis_exporter_test:9121' ]
        labels:
          env: test
          instance: redis-test
  - job_name: 'redis-uat'
    static_configs:
      - targets: [ 'redis_exporter_uat:9121' ]
        labels:
          env: uat
          instance: redis-uat
  - job_name: 'redis-prod'
    static_configs:
      - targets: [ 'redis_exporter_prod:9121' ]
        labels:
          env: prod
          instance: redis-prod

alerting:                         #Alertmanager相关的配置
  alertmanagers:
  - static_configs:
    - targets:
      - alertmanager:9093         #指定告警模块

rule_files:                      #告警规则文件, 可以使用通配符
  - "/etc/prometheus/rules/*.yml"
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

redis-rule.yml

groups:
- name: redisdown
  rules:
  - alert: RedisDown
    expr: redis_up == 0
    for: 1m
    labels:
      severity: Critical
    annotations:
      summary: "RedisDown instance {{ $labels.instance  }}"
      description: Redis instance is down on {{ $labels.instance }}\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
  - alert: Redis linked too many clients
    expr: redis_connected_clients / redis_config_maxclients * 100 > 80
    for: 1m
    labels:
      name: instance
      severity: Warning
    annotations:
      summary:  "Redis too many connections (> 80%) (instance {{ $labels.instance }})
      description: "More than 80% of Redis connections are in use on {{ $labels.instance }}\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

4 遇到的问题

当配置redis.addr时，如果连接的redis监听端口是6379，也即默认端口，如果配置成-redis.addr redis://10.10.51.94:6379会导致exporter无法连接上redis，可通过exporter的log查看，修改为-redis.addr redis://10.10.51.94可以连接成功。
该问题没有查清原因，通过在redis侧抓包看，exporter连接跟通过redis-cli连接发出的TCP SYN并没有不同，但前者的SYN被redis RST掉

在exporter所在主机连接redis的抓包

通过export连接redis，并配置成-redis.addr redis://10.10.51.94:6379时抓包