k8s 1.23.1 部署elk之elastalert日志系统错误日志钉钉报警，报警模板配置_elastalert 2无法使用内置钉钉报警

1. 安装efk日志系统，我是k8s安装的可以参考：
   https://blog.csdn.net/weixin_43606975/article/details/125060825?spm=1001.2014.3001.5501
2. k8s安装elastalert在这之前需要先做dockerfile文件：
   dockerfile文件目录：如下

#master.zip包
wget https://github.com/xuyaoqiang/elastalert-dingtalk-plugin/archive/master.zip
1
2

国内地址

#dingtalk_alert.py
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import json
import requests
from elastalert.alerts import Alerter, DateTimeEncoder
from requests.exceptions import RequestException
from elastalert.util import EAException
import time
import hmac
import hashlib
import base64
import urllib.parse

class DingTalkAlerter(Alerter):
    required_options = frozenset(['dingtalk_webhook', 'dingtalk_msgtype'])

    def __init__(self, rule):
        super(DingTalkAlerter, self).__init__(rule)
        self.dingtalk_webhook_url = self.rule['dingtalk_webhook']
        self.dingtalk_msgtype = self.rule.get('dingtalk_msgtype', 'text')
        self.dingtalk_isAtAll = self.rule.get('dingtalk_isAtAll', False)
        self.dingtalk_title = self.rule.get('dingtalk_title', '')
        self.dingtalk_secret = self.rule.get('dingtalk_secret','')
    def format_body(self, body):
        return body.encode('utf8')

    def alert(self, matches):
        headers = {
            "Content-Type": "application/json",
            "Accept": "application/json;charset=utf-8"
        }
        body = self.create_alert_body(matches)
        payload = {
            "msgtype": self.dingtalk_msgtype,
            "text": {
                "content": body
            },
            "at": {
                "isAtAll": False
            }
        }
        if self.dingtalk_secret!="":
            timestamp = str(round(time.time() * 1000))
            secret = self.dingtalk_secret
            secret_enc = secret.encode('utf-8')
            string_to_sign = '{}\n{}'.format(timestamp, secret)
            string_to_sign_enc = string_to_sign.encode('utf-8')
            hmac_code = hmac.new(secret_enc, string_to_sign_enc, digestmod=hashlib.sha256).digest()
            sign = urllib.parse.quote_plus(base64.b64encode(hmac_code))
            self.dingtalk_webhook_url=self.dingtalk_webhook_url+"&timestamp={}&sign={}".format(timestamp,sign)

        try:
            response = requests.post(self.dingtalk_webhook_url,
                                     data=json.dumps(payload, cls=DateTimeEncoder),
                                     headers=headers)
            response.raise_for_status()
        except RequestException as e:
            raise EAException("Error request to Dingtalk: {0}".format(str(e)))

    def get_info(self):
        return {
            "type": "dingtalk",
            "dingtalk_webhook": self.dingtalk_webhook_url
        }
        pass

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

#Dockerfile
FROM jertel/elastalert-docker:0.2.4
ADD master.zip  /opt/elastalert/
RUN cd /opt/elastalert;unzip master.zip;cd elastalert-dingtalk-plugin-master;pip3 install -i https://mirrors.aliyun.com/pypi/simple/   pyOpenSSL==16.2.0;pip3 install -i https://mirrors.aliyun.com/pypi/simple/  setuptools==46.1.3;cp -r elastalert_modules /usr/local/lib/python3.6/;cd  /usr/local/lib/python3.6/elastalert_modules; rm -rf dingtalk_alert.py
ADD dingtalk_alert.py /usr/local/lib/python3.6/elastalert_modules/
ENV TZ=Asia/Shanghai
RUN ln -snf /usr/share/zoneinfo/$TZ  /etc/localtime  && echo $TZ > /etc/timezone
1
2
3
4
5
6
7

docker 制作镜像：

docker build -t  my-elatrt:v22   ./
1

3. k8s安装elastalert

#cat elastalert.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: elastalert-config
  namespace: kube-logging
  labels:
    app: elastalert
data:
  elastalert_config: |-            #  elastalert配置文件
    ---
    rules_folder: /opt/rules       # 指定规则的目录
    scan_subdirectories: false
    es_host: elasticsearch 
    es_port: 9200
    run_every:                     # 多久从 ES 中查询一次
     seconds:  30              
    buffer_time:              #向上翻30分钟查找
      minutes: 30    
    writeback_index: elastalert    #创建索引名字
    use_ssl: False      #ssl不做认证
    verify_certs: True
    alert_time_limit:             # 失败重试限制
      minutes:  2400

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: elastalert-rules
  namespace: kube-logging
  labels:
    app: elastalert
data:
  rule_config.yaml: |-        # elastalert规则文件
    name: test-alert     # 规则名字，唯一值
    es_host: elasticsearch     #es地址，k8s的es
    es_port: 9200               #es端口
    type: any                  #所有类型
    index: k8s-*               #要搜索的索引
    num_events: 1  
    timeframe:
      minutes: 1   
     #1分钟内，统计个数大于等于1个触发报警
    filter:
     - query:
        query_string:
          query: "kubernetes.host:node1"  #key:value格式，匹配错误日志
    alert:
    - "elastalert_modules.dingtalk_alert.DingTalkAlerter"  #钉钉模块
    dingtalk_webhook: "https://oapi.dingtalk.com/robot/send?   access_token=4df2745e8df1de6d0429e35caf15e032e2b33ee2ba73899043c9995"  #钉钉地址
    dingtalk_sercurity_tpye: "sign"      #钉钉加签格式，感觉可以不要
    dingtalk_msgtype: "text"             #发消息内容
    dingtalk_secret: "SECe079af795abd316a7e1f431ee8ebcf082cc0b0611a859da37ec"  #钉钉加签
    alert_subject: "报错啦！！！"      #报警信息
    alert_text_type: alert_text_only 
    alert_text: |  #和下面匹配key:value
     日志监控
     time:{}
     hostname:{}
     error:{}
     mess:{}
    alert_text_args:
    - "@timestamp"
    - kubernetes.host
    - method
    - message
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: elastalert
  namespace: kube-logging
  labels:
    app: elastalert
spec:
  selector:
    matchLabels:
      app: elastalert
  template:
    metadata:
      labels:
        app: elastalert
    spec:
      containers:
      - name: elastalert
        image: my-elatrt:v22
        imagePullPolicy: IfNotPresent
        command: ["/opt/elastalert/run.sh"] 
        volumeMounts:
        - name: config
          mountPath: /opt/config
        - name: rules
          mountPath: /opt/rules
        resources:
          limits:
            cpu: 50m
            memory: 256Mi
          requests:
            cpu: 50m
            memory: 256Mi
      volumes:
      - name: rules
        configMap:
          name: elastalert-rules
      - name: config
        configMap:
          name: elastalert-config
          items:
          - key: elastalert_config
            path: elastalert_config.yaml


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

4. 启动

kubectl apply -f elastalert.yaml
1

5. 查看钉钉报警：

6. 错误收集
   遇到了很多问题：

7. 时间问题，由于elastalert时间一直没有正确导致一直都没有报警触发，所以一定要保证elastalert时间正确。
   

8. 刚开始没有dingidng模块，各种报错。

9. elastalert排错命令：

 #测试命令
 elastalert-test-rule   --config /opt/config/elastalert_config.yaml   /opt/rules/rule_config.yaml
 #启动命令，正常情况下不需要执行，容器自己执行了。
 python3 -m elastalert.elastalert --verbose --config /opt/config/elastalert_config.yaml --rule /app/elastalert/rule/nginx.yaml 
1
2
3
4

正常如下图：

你会在kibana里面看到这条信息说明就没问题。（要把这个索引加进去才能看到，不会自己出来的。）

9. kibana时间一定要正确不然报警的时间就是相差8小时：

新增：

多索引，多匹配：

name: MultipleErrorLogs
type: frequency
num_events: 5    
timeframe:      
  minutes: 10
#10分钟内，统计个数大于等于5个触发报警
index:
    - myapp-logs-*
    - otherapp-logs-*
filter:
- query:
    query_string:
      query: "error OR exception OR warning"
      default_operator: OR
- query:
    query_string:
      query: "something else"
      default_operator: OR
- query:
    query_string:
      query: "another error"
      default_operator: OR
alert:
- "email"
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

多个单索引，单匹配：

name: ErrorAlert1
type: frequency
num_events: 1
timeframe:
  minutes: 5
index: my-index-1
filter:
- query:
    query_string:
      query: "error OR exception"
      default_operator: OR
alert:
- "email"

name: ErrorAlert2
type: frequency
num_events: 1
timeframe:
  minutes: 5
index: my-index-2
filter:
- query:
    query_string:
      query: "file not found OR invalid argument"
      default_operator: OR
alert:
- "email"
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

新配置

apiVersion: v1
kind: ConfigMap
metadata:
  name: elastalert-config
  namespace: logging
  labels:
    app: elastalert
data:
  elastalert_config: |-            #  elastalert配置文件
    ---
    rules_folder: /opt/rules       # 指定规则的目录
    scan_subdirectories: false
    es_host: elasticsearch
    es_port: 9200
    run_every:                     # 多久从 ES 中查询一次
      minutes:  1
    buffer_time:              #向上翻30分钟查找
      minutes: 30
    writeback_index: elastalert    #创建索引名字
    use_ssl: False      #ssl不做认证
    verify_certs: True
    alert_time_limit:             # 失败重试限制
      minutes:  2400

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: elastalert-rules
  namespace:  logging
  labels:
    app: elastalert
data:
  rule_config.yaml: |-        # elastalert规则文件
     name: test-alert     # 规则名字，唯一值
     es_host: elasticsearch     #es地址，k8s的es
     es_port: 9200               #es端口
     #type: any                  #所有类型
     index: checknewgoodssign-*           #要搜索的索引
     #策略规则，如果在10分钟内，匹配的个数大于等于5，那么就触发钉钉报警
     type: frequency
     num_events: 5
     timeframe:
       minutes: 10
     filter:
      - query:
         query_string:
           query: "\"获取AccessToken失败\""
     alert:
     - "elastalert_modules.dingtalk_alert.DingTalkAlerter"  #钉钉模块
     dingtalk_webhook: "https://oapi.dingtalk.com/robot/send?access_token=cc67ce67cb556271e15f5e7f44908da113c2d30f98e4647e704"  #钉钉地址
     dingtalk_sercurity_tpye: "sign"      #钉钉加签格式，感觉可以不要
     dingtalk_msgtype: "text"             #发消息内容
     dingtalk_secret: "SEC2dc9489ccac40fb1ee02d13844b066c6fd8fd54ffcaa834df2"  #钉钉加签
     alert_subject: "报错啦！！！"      #报警信息
     alert_text_type: alert_text_only
     alert_text: |  #和下面匹配key:value
      日志监控
      mess: {}
      pod-name: {}
     alert_text_args:
     - message
     - kubernetes.pod.name


  rule_kafka.yaml: |-        # elastalert规则文件
    name: kafka-alert     # 规则名字，唯一值
    es_host: elasticsearch     #es地址，k8s的es
    es_port: 9200               #es端口
    #type: any                  #所有类型
    index: dtk-go-tb-order-*           #要搜索的索引
    type: frequency
    num_events: 5
    timeframe:
      minutes: 10
    filter:
     - query:
        query_string:
          query: "\"写入Kafka消息彻底失败\""
    alert:
    - "elastalert_modules.dingtalk_alert.DingTalkAlerter"  #钉钉模块
    dingtalk_webhook: "https://oapi.dingtalk.com/robot/send?access_token=cc67ce67cb556271e15f5e7f44908da113c2d30ea521cf7e704"  #钉钉地址
    dingtalk_sercurity_tpye: "sign"      #钉钉加签格式，感觉可以不要
    dingtalk_msgtype: "text"             #发消息内容
    dingtalk_secret: "SEC2dc9489ccac40fb1ee02d13844b066c6fd8fd4ffcaa834df2"  #钉钉加签
    alert_subject: "报错啦！！！"      #报警信息
    alert_text_type: alert_text_only
    alert_text: |  #和下面匹配key:value
     日志监控
     mess: {}
     pod-name: {}
    alert_text_args:
    - message
    - kubernetes.pod.name



#正则
  rule_config.yaml: |-        # elastalert规则文件
     name: test-alert     # 规则名字，唯一值
     es_host: elasticsearch     #es地址，k8s的es
     es_port: 9200               #es端口
     #type: any                  #所有类型
     index: dtk-go-taobao-api-*           #要搜索的索引
     type: frequency
     num_events: 1
     timeframe:
       minutes: 2
     filter:
      - query:
         regexp:
           message: "Error \\d{4} \\(|panic:"
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112