\u4e4b\u524d\u7528 Python \u5b9e\u73b0\u4e86\u4e00\u4e2a\u975e\u5e38\u7b80\u964b\u7684 AlertManager \u7684\u9489\u9489\u63a5\u6536\u5668\uff0c\u4e00\u76f4\u60f3\u5728\u9489\u9489\u7684\u6d88\u606f\u901a\u77e5\u4e2d\u5c06\u5f53\u524d\u62a5\u8b66\u56fe\u8868\u4e5f\u5c55\u793a\u51fa\u6765\uff0c\u8fd9\u6837\u663e\u7136\u5bf9\u7528\u6237\u6765\u8bf4\u66f4\u52a0\u53cb\u597d\u3002\u4e4b\u524d\u60f3\u7684\u601d\u8def\u662f\u901a\u8fc7\u722c\u866b\u7684\u65b9\u5f0f\u53bb Prometheus \u9875\u9762\u5c06 Graph \u56fe\u5f62\u622a\u56fe\u4fdd\u5b58\u4e0b\u6765\uff0c\u8be5\u65b9\u5f0f\u7406\u8bba\u4e0a\u786e\u5b9e\u662f\u53ef\u884c\u7684\uff0c\u4f46\u662f\u8fd9\u79cd\u65b9\u5f0f\u4e0d\u7a33\u5b9a\u56e0\u7d20\u8f83\u591a\uff0c\u800c\u4e14\u4f1a\u5360\u7528\u5927\u91cf\u7684\u8d44\u6e90\u3002<\/p>\n

\u4eca\u5929\u6362\u4e86\u53e6\u5916\u4e00\u79cd\u65b9\u5f0f\u6765\u5b9e\u73b0\uff0c\u76f4\u63a5\u53bb\u7ed8\u5236\u6e32\u67d3\u62a5\u8b66\u56fe\u8868\uff0c\u7136\u540e\u4e0a\u4f20\u5230\u5bf9\u8c61\u5b58\u50a8\u4e2d\u4fdd\u5b58\u8d77\u6765\uff0c\u5728\u9489\u9489\u4e2d\u5c31\u53ef\u4ee5\u76f4\u63a5\u5c55\u793a\u4e86\uff0cPromoter \u5c31\u662f\u8fd9\u4e2a\u65b9\u6848\u7684\u4e00\u4e2a\u5b9e\u73b0\uff0c\u652f\u6301\u5728\u6d88\u606f\u901a\u77e5\u4e2d\u5c55\u793a\u5b9e\u65f6\u62a5\u8b66\u56fe\u8868\uff0c\u6548\u679c\u56fe\u5982\u4e0b\u6240\u793a\uff1a<\/p>\n

\u76ee\u524d\u662f\u5c06\u62a5\u8b66\u6570\u636e\u6e32\u67d3\u6210\u56fe\u7247\u540e\u4e0a\u4f20\u5230 S3 \u5bf9\u8c61\u5b58\u50a8\uff0c\u6240\u4ee5\u9700\u8981\u914d\u7f6e\u4e00\u4e2a\u5bf9\u8c61\u5b58\u50a8(\u963f\u91cc\u4e91 OSS \u4e5f\u53ef\u4ee5)\uff0c\u6b64\u5916\u6d88\u606f\u901a\u77e5\u5c55\u793a\u6837\u5f0f\u652f\u6301\u6a21\u677f\u5b9a\u5236.<\/p>\n

\u6a21\u677f<\/strong><\/span><\/div>\n
\u9ed8\u8ba4\u6a21\u677f\u4f4d\u4e8e template\/default.tmpl\uff0c\u53ef\u4ee5\u6839\u636e\u81ea\u5df1\u9700\u6c42\u5b9a\u5236\uff1a<\/p>\n
{{ define \"__subject\" }}[{{ .Status | toUpper }}{{ if eq .Status \"firing\" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join \" \" }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join \" \" }}{{ end }}){{ end }}{{ end }}\r\n{{ define \"__alertmanagerURL\" }}{{ .ExternalURL }}\/#\/alerts?receiver={{ .Receiver }}{{ end }}\r\n{{ define \"default.__text_alert_list\" }}{{ range . }}\r\n### {{ .Annotations.summary }}\r\n**\u8be6\u60c5:** {{ .Annotations.description }}\r\n{{ range .Images }}\r\n**\u6761\u4ef6:** `{{ .Title }}`\r\n![📈]({{ .Url }})\r\n{{- end }}\r\n**\u6807\u7b7e:**\r\n{{ range .Labels.SortedPairs }}{{ if and (ne (.Name) \"severity\") (ne (.Name) \"summary\") }}> - {{ .Name }}: {{ .Value | markdown | html }}\r\n{{ end }}{{ end }}\r\n{{ end }}{{ end }}\r\n{{\/* Default *\/}}\r\n{{ define \"default.title\" }}{{ template \"__subject\" . }}{{ end }}\r\n{{ define \"default.content\" }}\r\n{{ if gt (len .Alerts.Firing) 0 -}}\r\n#### **{{ .Alerts.Firing | len }} \u6761\u62a5\u8b66**\r\n{{ template \"default.__text_alert_list\" .Alerts.Firing }}\r\n{{ range .AtMobiles }}@{{ . }}{{ end }}\r\n{{- end }}\r\n{{ if gt (len .Alerts.Resolved) 0 -}}\r\n#### **{{ .Alerts.Resolved | len }} \u6761\u62a5\u8b66\u6062\u590d**\r\n{{ template \"default.__text_alert_list\" .Alerts.Resolved }}\r\n{{ range .AtMobiles }}@{{ . }}{{ end }}\r\n{{- end }}\r\n{{- end }}<\/pre>\n
\u90e8\u7f72<\/strong><\/span><\/div>\n
\u9ed8\u8ba4\u914d\u7f6e\u6587\u4ef6\u5982\u4e0b\u6240\u793a\uff0c\u653e\u7f6e\u5728 \/etc\/promoter\/config.yaml\uff1a<\/p>\n
debug: true\r\nhttp_port: 8080\r\ntimeout: 5s\r\nprometheus_url: # Prometheus \u7684\u5730\u5740\r\nmetric_resolution: 100\r\ns3:\r\n access_key: \r\n secret_key: \r\n endpoint: oss-cn-beijing.aliyuncs.com\r\n region: cn-beijing\r\n bucket: \r\ndingtalk:\r\n url: https:\/\/oapi.dingtalk.com\/robot\/send?access_token=\r\n secret: # secret for signature\r\n<\/sec><\/token><\/bucket><\/sk><\/ak><\/prometheus_url><\/pre>\n
\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528 Docker \u955c\u50cf cnych\/promoter:v0.1.1 \u90e8\u7f72\uff0c\u5728 Kubernetes \u4e2d\u90e8\u7f72\u53ef\u4ee5\u76f4\u63a5\u53c2\u8003 deploy\/kubernetes\/promoter.yaml\u3002<\/p>\n
\u542f\u52a8\u5b8c\u6210\u540e\u5728 AlertManager \u914d\u7f6e\u4e2d\u6307\u5b9a Webhook \u5730\u5740\u5373\u53ef\uff1a<\/p>\n
route:\r\n group_by: ['alertname', 'cluster']\r\n group_wait: 30s\r\n group_interval: 2m\r\n repeat_interval: 1h\r\n receiver: webhook\r\nreceivers:\r\n- name: 'webhook'\r\n webhook_configs:\r\n - url: 'http:\/\/promoter.kube-mon.svc.cluster.local:8080\/webhook' # \u914d\u7f6e promoter \u7684 webhook \u63a5\u53e3\r\n send_resolved: true\r\n<\/pre>\n
\u6838\u5fc3\u539f\u7406<\/strong><\/span><\/div>\n
\u8be5\u9879\u76ee\u91c7\u7528 golang \u5b9e\u73b0\uff0cWebhook \u7684\u5b9e\u73b0\u5f88\u7b80\u5355\uff0c\u8fd9\u91cc\u7684\u6838\u5fc3\u90e8\u5206\u662f\u5982\u4f55\u6e32\u67d3\u76d1\u63a7\u56fe\u8868\uff0c\u6838\u5fc3\u65b9\u5f0f\u662f\u901a\u8fc7 Prometheus \u7684 API \u63a5\u53e3\u6765\u83b7\u53d6\u67e5\u8be2\u7684\u6307\u6807\u6570\u636e\uff1a<\/p>\n
func Metrics(server, query string, queryTime time.Time, duration, step time.Duration) (promModel.Matrix, error) {\r\n client, err := prometheus.NewClient(prometheus.Config{Address: server})\r\n if err != nil {\r\n return nil, fmt.Errorf(\"failed to create Prometheus client: %v\", err)\r\n }\r\n api := prometheusApi.NewAPI(client)\r\n value, _, err := api.QueryRange(context.Background(), query, prometheusApi.Range{\r\n Start: queryTime.Add(-duration),\r\n End: queryTime,\r\n Step: duration \/ step,\r\n })\r\n if err != nil {\r\n return nil, fmt.Errorf(\"failed to query Prometheus: %v\", err)\r\n }\r\n metrics, ok := value.(promModel.Matrix)\r\n if !ok {\r\n return nil, fmt.Errorf(\"unsupported result format: %s\", value.Type().String())\r\n }\r\n return metrics, nil\r\n}\r\n<\/pre>\n
\u7136\u540e\u5c06\u83b7\u53d6\u7684\u6307\u6807\u7ed8\u5236\u51fa\u6765\uff0c\u56fe\u5f62\u7ed8\u5236\u4f7f\u7528\u7684 gonum.org\/v1\/plot \u8fd9\u4e2a\u5305\u6765\u5b9e\u73b0\u7684\uff1a<\/p>\n
func PlotMetric(metrics promModel.Matrix, level float64, direction string) (io.WriterTo, error) {\r\n p, err := plot.New()\r\n if err != nil {\r\n return nil, fmt.Errorf(\"failed to create new plot: %v\", err)\r\n }\r\n textFont, err := vg.MakeFont(\"Helvetica\", 3*vg.Millimeter)\r\n if err != nil {\r\n return nil, fmt.Errorf(\"failed to load font: %v\", err)\r\n }\r\n evalTextFont, err := vg.MakeFont(\"Helvetica\", 5*vg.Millimeter)\r\n if err != nil {\r\n return nil, fmt.Errorf(\"failed to load font: %v\", err)\r\n }\r\n evalTextStyle := draw.TextStyle{\r\n Color: color.NRGBA{A: 150},\r\n Font: evalTextFont,\r\n XAlign: draw.XRight,\r\n YAlign: draw.YBottom,\r\n }\r\n p.X.Tick.Marker = plot.TimeTicks{Format: \"15:04:05\"}\r\n p.X.Tick.Label.Font = textFont\r\n p.Y.Tick.Label.Font = textFont\r\n p.Legend.Font = textFont\r\n p.Legend.Top = true\r\n p.Legend.YOffs = 15 * vg.Millimeter\r\n \/\/ Color palette for drawing lines\r\n paletteSize := 8\r\n palette, err := brewer.GetPalette(brewer.TypeAny, \"Dark2\", paletteSize)\r\n if err != nil {\r\n return nil, fmt.Errorf(\"failed to get color palette: %v\", err)\r\n }\r\n colors := palette.Colors()\r\n var lastEvalValue float64\r\n for s, sample := range metrics {\r\n data := make(plotter.XYs, 0)\r\n for _, v := range sample.Values {\r\n fs := v.Value.String()\r\n if fs == \"NaN\" {\r\n _, err := drawLine(data, colors, s, paletteSize, p, metrics, sample)\r\n if err != nil {\r\n return nil, err\r\n }\r\n data = make(plotter.XYs, 0)\r\n continue\r\n }\r\n f, err := strconv.ParseFloat(fs, 64)\r\n if err != nil {\r\n return nil, fmt.Errorf(\"sample value not float: %s\", v.Value.String())\r\n }\r\n data = append(data, plotter.XY{X: float64(v.Timestamp.Unix()), Y: f})\r\n lastEvalValue = f\r\n }\r\n _, err := drawLine(data, colors, s, paletteSize, p, metrics, sample)\r\n if err != nil {\r\n return nil, err\r\n }\r\n }\r\n var polygonPoints plotter.XYs\r\n if direction == \"<\" {\r\n polygonPoints = plotter.XYs{{X: p.X.Min, Y: level}, {X: p.X.Max, Y: level}, {X: p.X.Max, Y: p.Y.Min}, {X: p.X.Min, Y: p.Y.Min}}\r\n } else {\r\n polygonPoints = plotter.XYs{{X: p.X.Min, Y: level}, {X: p.X.Max, Y: level}, {X: p.X.Max, Y: p.Y.Max}, {X: p.X.Min, Y: p.Y.Max}}\r\n }\r\n poly, err := plotter.NewPolygon(polygonPoints)\r\n if err != nil {\r\n return nil, err\r\n }\r\n poly.Color = color.NRGBA{R: 255, A: 40}\r\n poly.LineStyle.Color = color.NRGBA{R: 0, A: 0}\r\n p.Add(poly)\r\n p.Add(plotter.NewGrid())\r\n \/\/ Draw plot in canvas with margin\r\n margin := 6 * vg.Millimeter\r\n width := 20 * vg.Centimeter\r\n height := 10 * vg.Centimeter\r\n c, err := draw.NewFormattedCanvas(width, height, \"png\")\r\n if err != nil {\r\n return nil, fmt.Errorf(\"failed to create canvas: %v\", err)\r\n }\r\n cropedCanvas := draw.Crop(draw.New(c), margin, -margin, margin, -margin)\r\n p.Draw(cropedCanvas)\r\n \/\/ Draw last evaluated value\r\n evalText := fmt.Sprintf(\"latest evaluation: %.2f\", lastEvalValue)\r\n plotterCanvas := p.DataCanvas(cropedCanvas)\r\n trX, trY := p.Transforms(&plotterCanvas)\r\n evalRectangle := evalTextStyle.Rectangle(evalText)\r\n points := []vg.Point{\r\n {X: trX(p.X.Max) + evalRectangle.Min.X - 8*vg.Millimeter, Y: trY(lastEvalValue) + evalRectangle.Min.Y - vg.Millimeter},\r\n {X: trX(p.X.Max) + evalRectangle.Min.X - 8*vg.Millimeter, Y: trY(lastEvalValue) + evalRectangle.Max.Y + vg.Millimeter},\r\n {X: trX(p.X.Max) + evalRectangle.Max.X - 6*vg.Millimeter, Y: trY(lastEvalValue) + evalRectangle.Max.Y + vg.Millimeter},\r\n {X: trX(p.X.Max) + evalRectangle.Max.X - 6*vg.Millimeter, Y: trY(lastEvalValue) + evalRectangle.Min.Y - vg.Millimeter},\r\n }\r\n plotterCanvas.FillPolygon(color.NRGBA{R: 255, G: 255, B: 255, A: 90}, points)\r\n plotterCanvas.FillText(evalTextStyle, vg.Point{X: trX(p.X.Max) - 6*vg.Millimeter, Y: trY(lastEvalValue)}, evalText)\r\n return c, nil\r\n}\r\nfunc drawLine(data plotter.XYs, colors []color.Color, s int, paletteSize int, p *plot.Plot, metrics promModel.Matrix, sample *promModel.SampleStream) (*plotter.Line, error) {\r\n var l *plotter.Line\r\n var err error\r\n if len(data) > 0 {\r\n l, err = plotter.NewLine(data)\r\n if err != nil {\r\n return &plotter.Line{}, fmt.Errorf(\"failed to create line: %v\", err)\r\n }\r\n l.LineStyle.Width = vg.Points(1)\r\n l.LineStyle.Color = colors[s%paletteSize]\r\n p.Add(l)\r\n if len(metrics) > 1 {\r\n m := labelText.FindStringSubmatch(sample.Metric.String())\r\n if m != nil {\r\n p.Legend.Add(m[1], l)\r\n }\r\n }\r\n }\r\n return l, nil\r\n}<\/pre>\n","protected":false},"excerpt":{"rendered":"
\u4e4b\u524d\u7528 Python \u5b9e\u73b0\u4e86\u4e00\u4e2a\u975e\u5e38\u7b80\u964b\u7684 AlertManager \u7684\u9489\u9489\u63a5\u6536\u5668\uff0c\u4e00\u76f4\u60f3\u5728\u9489\u9489\u7684\u6d88\u606f\u901a\u77e5\u4e2d\u5c06 […]<\/p>\n","protected":false},"author":362,"featured_media":79710,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_acf_changed":false,"footnotes":""},"categories":[55],"tags":[],"class_list":["post-236718","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-thread"],"acf":[],"_links":{"self":[{"href":"https:\/\/lrxjmw.cn\/wp-json\/wp\/v2\/posts\/236718","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/lrxjmw.cn\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/lrxjmw.cn\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/lrxjmw.cn\/wp-json\/wp\/v2\/users\/362"}],"replies":[{"embeddable":true,"href":"https:\/\/lrxjmw.cn\/wp-json\/wp\/v2\/comments?post=236718"}],"version-history":[{"count":3,"href":"https:\/\/lrxjmw.cn\/wp-json\/wp\/v2\/posts\/236718\/revisions"}],"predecessor-version":[{"id":236722,"href":"https:\/\/lrxjmw.cn\/wp-json\/wp\/v2\/posts\/236718\/revisions\/236722"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/lrxjmw.cn\/wp-json\/wp\/v2\/media\/79710"}],"wp:attachment":[{"href":"https:\/\/lrxjmw.cn\/wp-json\/wp\/v2\/media?parent=236718"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/lrxjmw.cn\/wp-json\/wp\/v2\/categories?post=236718"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/lrxjmw.cn\/wp-json\/wp\/v2\/tags?post=236718"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}