prometheus黑盒监控

琉璃1年前技术文章481

一.背景

黑盒监控:主要关注的现象,一般都是正在发生的东西,例如出现一个告警,业务接口不正常,那么这种监控就是站在用户的角度能看到的监控,重点在于能对正在发生的故障进行告警。

二.操作前了解相关配置和要求

下述内容基于helm和k8s中的prometheus监控,如果环境不一样需要根据实际情况进行调整

三.操作步骤

详细的操作步骤说明

helm安装blackbox_exporter

获取helm仓库信息

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update



安装Chart包

helm install [RELEASE_NAME] prometheus-community/prometheus-blackbox-exporter



卸载Chart包

helm uninstall [RELEASE_NAME]



安装完成后可以看到如下资源

配置blackbox_ exporter

配置blackbox_exporter配置文件

首先我们要配置我们的blackbox_ exporter,blackbox_exporter的配置文件为blackbox.yml。当我们以helm方式安装blackbox_ exporter时,配置文件是以configMap的形式存在的。查看下我们的配置文件。

kubectl get configMap -n prometheus

kubectl get configMap -n prometheus prometheus-blackbox-exporter



官方配置文件样例

这里提供一个官方的样例配置文件,根据我们的需求修改配置文件即可。

modules:
  http_2xx_example:
    prober: http
    timeout: 5s
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: []  # Defaults to 2xx
      method: GET
      headers:
        Host: vhost.example.com
        Accept-Language: en-US
        Origin: example.com
      no_follow_redirects: false
      fail_if_ssl: false
      fail_if_not_ssl: false
      fail_if_body_matches_regexp:
        - "Could not connect to database"
      fail_if_body_not_matches_regexp:
        - "Download the latest version here"
      fail_if_header_matches: # Verifies that no cookies are set
        - header: Set-Cookie
          allow_missing: true
          regexp: '. *'
      fail_if_header_not_matches:
        - header: Access-Control-Allow-Origin
          regexp: '(\*|example\.com)'
      tls_config:
        insecure_skip_verify: false
      preferred_ip_protocol: "ip4" # defaults to "ip6"
      ip_protocol_fallback: false  # no fallback to "ip6"
  http_post_2xx:
    prober: http
    timeout: 5s
    http:
      method: POST
      headers:
        Content-Type: application/json
      body: '{}'
  http_basic_auth_example:
    prober: http
    timeout: 5s
    http:
      method: POST
      headers:
        Host: "login.example.com"
      basic_auth:
        username: "username"
        password: "mysecret"
  http_custom_ca_example:
    prober: http
    http:
      method: GET
      tls_config:
        ca_file: "/certs/my_cert.crt"
  http_gzip:
    prober: http
    http:
      method: GET
      compression: gzip
  http_gzip_with_accept_encoding:
    prober: http
    http:
      method: GET
      compression: gzip
      headers:
        Accept-Encoding: gzip
  tls_connect:
    prober: tcp
    timeout: 5s
    tcp:
      tls: true
  tcp_connect_example:
    prober: tcp
    timeout: 5s
  imap_starttls:
    prober: tcp
    timeout: 5s
    tcp:
      query_response:
        - expect: "OK.*STARTTLS"
        - send: ". STARTTLS"
        - expect: "OK"
        - starttls: true
        - send: ". capability"
        - expect: "CAPABILITY IMAP4rev1"
  smtp_starttls:
    prober: tcp
    timeout: 5s
    tcp:
      query_response:
        - expect: "^220 ([^ ]+) ESMTP (.+)$"
        - send: "EHLO prober\r"
        - expect: "^250-STARTTLS"
        - send: "STARTTLS\r"
        - expect: "^220"
        - starttls: true
        - send: "EHLO prober\r"
        - expect: "^250-AUTH"
        - send: "QUIT\r"
  irc_banner_example:
    prober: tcp
    timeout: 5s
    tcp:
      query_response:
        - send: "NICK prober"
        - send: "USER prober prober prober :prober"
        - expect: "PING :([^ ]+)"
          send: "PONG ${1}"
        - expect: "^:[^ ]+ 001"
  icmp_example:
    prober: icmp
    timeout: 5s
    icmp:
      preferred_ip_protocol: "ip4"
      source_ip_address: "127.0.0.1"
  dns_udp_example:
    prober: dns
    timeout: 5s
    dns:
      query_name: "www.prometheus.io"
      query_type: "A"
      valid_rcodes:
      - NOERROR
      validate_answer_rrs:
        fail_if_matches_regexp:
        - ".*127.0.0.1"
        fail_if_all_match_regexp:
        - ".*127.0.0.1"
        fail_if_not_matches_regexp:
        - "www.prometheus.io.\t300\tIN\tA\t127.0.0.1"
        fail_if_none_matches_regexp:
        - "127.0.0.1"
      validate_authority_rrs:
        fail_if_matches_regexp:
        - ".*127.0.0.1"
      validate_additional_rrs:
        fail_if_matches_regexp:
        - ".*127.0.0.1"
  dns_soa:
    prober: dns
    dns:
      query_name: "prometheus.io"
      query_type: "SOA"
  dns_tcp_example:
    prober: dns
    dns:
      transport_protocol: "tcp" # defaults to "udp"
      preferred_ip_protocol: "ip4" # defaults to "ip6"
      query_name: "www.prometheus.io"



http_2xx模块

简单看一下http_2xx这个模块

modules:
  http_2xx:
    prober: http
    timeout: 5s
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: [200]  # Defaults to 2xx
      method: GET
      headers:
        Host: blackbox.ilomumu.xyz
        Accept-Language: en-US
        Origin: ilomumu.xyz
      # 不跟重定向跟随
      no_follow_redirects: false
      # 使用ssl时判断为失败
      fail_if_ssl: false
      # 不使用ssl时判断为失败
      fail_if_not_ssl: false
      # body内容匹配就判断为失败
      fail_if_body_matches_regexp:
        - "Could not connect to database"
      fail_if_body_not_matches_regexp:
        - "Download the latest version here"
      # header匹配则判断为失败
      fail_if_header_matches: # Verifies that no cookies are set
        - header: Set-Cookie
          allow_missing: true
          regexp: '.*'
      # header不匹配则判断为失败
      fail_if_header_not_matches:
        - header: Access-Control-Allow-Origin
          regexp: '(\ *|example\.com)'
      # tls 证书配置
      tls_config:
        insecure_skip_verify: false
      preferred_ip_protocol: "ip4" # defaults to "ip6"
      ip_protocol_fallback: false  # no fallback to "ip6"



我们使用的配置文件

modules:
  http_2xx:
    prober: http
    timeout: 5s
    http:
      valid_http_versions:
        - HTTP/1.1
        - HTTP/2.0
      valid_status_codes:
        - 200
      method: GET
      headers:
        Host: blackbox.ilomumu.xyz
        Accept-Language: en-US
        Origin: ilomumu.xyz
      # 这里没有使用no_follow_redirects是因为no_follow_redirects在未来的版本中会被follow_redirects替代
      follow_redirects: true
      fail_if_ssl: false
      fail_if_not_ssl: false
      tls_config:
        insecure_skip_verify: true
      preferred_ip_protocol: ip4
      ip_protocol_fallback: false


修改完需要重新部署deployment使新的配置文件生效

配置prometheus

配置blackbox_exporter到prometheus

这里如果要配置blackbox_exporter到二进制部署的prometheus很简单,我们只需要修改promethues的prometheus.yml文件,在scrape_configs配置相应字段即可。

scrape_configs:
  - job_name: 'blackbox'
    metrics_path: /probe
    params:
      module: [http_2xx]  # Look for a HTTP 200 response.
    static_configs:
      - targets:
        - http://prometheus.io    # Target to probe with http.
        - https://prometheus.io   # Target to probe with https.
        - http://example.com:8080 # Target to probe with http on port 8080.
    relabel_configs:
      - source_labels: [__address__]
        target_label:__ param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port.



根据我们的需求修改上述文件即可

配置blackbox_exporter到prometheus operator

编写ServiceMonitor文件

但是我们使用的是prometheus operator,此时我们如果直接进入容器修改prometheus.yml显然是不现实的。但是如果我们去查看configMap会发现也没有prometheus.yml相关的配置文件,那我们如何修改prometheus.yml?

这里其实我们要使用到ServiceMonitor这种资源,通过ServiceMonitor来使用prometheus的动态发现。

kubectl get ServiceMonitor -A



所以我们要编写一个适合我们使用的ServiceMonitor来讲blackbox_exporter配置到promethues中

接下来的文件中relabelings段可能比较难以理解,下文会解释为何如此进行编写

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: prometheus-blackbox-exporter
  namespace: prometheus
spec:
  endpoints:
  - params:
      # 监控使用的module,这里就是我们blackbox.yml中配置的监控规则
      module:
      - http_2xx
    path: /probe # 此处路径固定,为blackbox-exporter暴露的监控页面
    port: web # 监控使用的端口(可以是service端口的名称)
    relabelings:
    # 正则替换这里主要是为了实现自定义监控路径和地址
    # 整体这段替换的目的就是为了将默认的目标地址替换为我们真正需要监控的地址和路径
    - action: replace
      # 替换内容
      regex: (. *)
      # 替换规则
      replacement: ${1}${2}
      # 分隔符
      separator: /
      # 替换的内容
      # 这里使用了两个内置变量
      sourceLabels:
      # 地址(需要监控的svc的地址)
      - __address__
      # 标签 prometheus.io.http.probe.path 的值
      - __meta_kubernetes_service_label_prometheus_io_http_probe_path
      # 被替换内容
      targetLabel:__ param_target
    # 这段替换的目的是为了将默认访问地址换成我们blackbox_exporter的地址
    - action: replace
      regex: (. *)
      # blackbox-exporter的svc地址
      # 为了实现跨namespace访问,地址需要写成{SERVICE_NAME}.{NAMESPACE_NAME}.svc.cluster.local 
      replacement: prometheus-blackbox-exporter.prometheus.svc.cluster.local:9115
      sourceLabels:
      - __address__
      targetLabel: __address__
    # 添加label方便后续进行辨认
    - action: replace
      regex: (.*)
      replacement: $1
      sourceLabels:
      -__ param_target
      targetLabel: instance
    # 添加label方便后续进行辨认
    - action: replace
      regex: (. *)
      replacement: $1
      sourceLabels:
      - __param_module
      targetLabel: module
  # 命名空间限制
  namespaceSelector:
    # 允许所有命名空间
    any: true
  # label匹配,通过该字段可以控制那些svc需要被监控
  selector:
    matchLabels:
      blackbox-monitor: "true"



[重要]relabelings为何如此配置

首先我们要了解黑盒监控的根本原理。

首先根据前文的介绍我们可以知道,黑盒监控简单来说就是以用户的身份测试服务的外部可见性。

那这里的用户是谁呢?

其实我们的blackbox_exporter就扮演了用户的身份。也就是说blackbox_exporter启动的pod去访问了我们的监控目标,blackbox的pod通过自己访问各个业务的结果生成了相关的监控结果。

所以说如果promethues想要获取监控指标和结果,是需要访问我们blackbox_exporter这个pod提供的页面来拿到相关的数据的。

例如:

📌我们需要使用blackbox_export中的module http_2xx来监控www.baidu.com。那么我们应该这样去访问这样的地址

http://blackbox_exporter:9115/probe?module=http_2xx&target=www.baidu.com



📌这里我们可以看出来我们需要监控的地址和module都是以参数的形式添加到了url中,而我们访问的目的还是blackbox_exporter的9115端口路径为probe

但是如果我们不配置relabelings对一些字段进行替换,根据默认的配置(prometheus.yml中scrape_configs)promethues回去访问需要监控的各个svc的/probe这个路径来获取数据。也就是说我们在prometheus targets页面上看到的endpoint是下面这个地址

# 此处为模拟的url地址,promethues targets页面上并不会显示url的参数部分,而是以另一种形式来展示
http://<需要监控svc地址>:9115/probe?module=<监控使用的模块>&target=<需要监控svc的地址>

# promethues targets页面上显示的方式
endpoint:http://<blackbox_exporter>:9115/probe
module=<监控使用的模块>
target=<需要监控svc的地址>

# 以监控www.baidu.com为例
http://www.baidu.com/probe?module=http_2xx&target=www.baidu.com



而我们真正需要访问的endpoint应该是这样的

# 此处为模拟的url地址,promethues targets页面上并不会显示url的参数部分,而是以另一种形式来展示
http://<blackbox_exporter>:9115/probe?module=<监控使用的模块>&target=<需要监控svc的地址>

# promethues targets页面上显示的方式
endpoint:http://<blackbox_exporter>:9115/probe
module=<监控使用的模块>
target=<需要监控svc的地址>

# 以监控www.baidu.com为例
http://blackbox_exporter:9115/probe?module=http_2xx&target=www.baidu.com



上述整个relabelings段存在的意义就是为了把endpint换成我们需要的endpoint

配置需要监控的svc

根据我们上面的ServiceMonitor文件可以得知,我们需要配置两个标签来实现让promethues发现我们需要监控的svc

# 以下均为标签

# 这个标签用来决定是否启用blackbox监控
blackbox-monitor: "true"

# 这个标签用来决定我们需要监控的健康检查路径(http://address:port/healthz)
# 路径前不需要添加'/'
# 如果没有健康检查路径可以不填写,默认会以根目录作为健康检查路径
prometheus.io/http-probe-path: healthz



这里需要注意又有lable中的值不允许出现/ ,所以如果我们的健康检查路径如果有多级,就无法使用lable来实现。

那么如果我们的健康检查路径中有多级可以使用注释(annotations)来实现。

使用annotations实现多级健康检查路径

修改ServiceMonitor文件

# 部分ServiceMonitior文件
    - action: replace
      # 替换内容
      regex: (. *)
      # 替换规则
      replacement: ${1}${2}
      # 分隔符
      separator: /
      # 替换的内容
      # 这里使用了两个内置变量
      sourceLabels:
      # 地址(需要监控的svc的地址)
      - __address__
      # 标签 prometheus.io.http.probe.path 的值
      # - __meta_kubernetes_service_label_prometheus_io_http_probe_path
      # 这里我们获取annotations的值而并非lable的值
      -__ meta_kubernetes_service_annotations_prometheus_io_http_probe_path
      # 被替换内容
      targetLabel:__ param_target



同时我们svc需要配置的内容也要变化

# 标签
# 这个标签用来决定是否启用blackbox监控
blackbox-monitor: "true"

# 注释
# 这个注释用来决定我们需要监控的健康检查路径(http://address:port/check/healthz)
# 路径前不需要添加'/'
# 如果没有健康检查路径可以不填写,默认会以根目录作为健康检查路径
prometheus.io/http-probe-path: check/healthz


查看promethues数据

上述配置完成后我们就可以在promethues中查看数据了

image.png


image.png



grafana仪表盘

image.png

仪表盘json文件

{
  "annotations": {
    "list": [
      {
        "$$hashKey": "object:20",
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      }
    ]
  },
  "description": "",
  "editable": true,
  "gnetId": 9965,
  "graphTooltip": 0,
  "id": 38,
  "iteration": 1650958653934,
  "links": [
    {
      "asDropdown": false,
      "icon": "bolt",
      "includeVars": false,
      "keepTime": false,
      "tags": [],
      "targetBlank": true,
      "title": "Update",
      "tooltip": "",
      "type": "link",
      "url": "https://grafana.com/dashboards/9965"
    },
    {
      "asDropdown": false,
      "icon": "question",
      "includeVars": false,
      "keepTime": false,
      "tags": [],
      "targetBlank": true,
      "title": "GitHub",
      "tooltip": "",
      "type": "link",
      "url": "https://github.com/starsliao/ConsulManager"
    }
  ],
  "panels": [
    {
      "collapsed": false,
      "datasource": {
        "type": "prometheus",
        "uid": "Prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 2422,
      "panels": [],
      "title": "总览",
      "type": "row"
    },
    {
      "datasource": null,
      "description": "注意:对于有重定向的URL,如果重定向后的域名与检查的域名不同,则检查的证书信息为重定向后的域名证书信息。如果需要检查源域名信息,请在blackbox中增加禁止重定向参数。",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "center",
            "displayMode": "auto",
            "filterable": false
          },
          "decimals": 2,
          "mappings": [],
          "noValue": "无",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "公司"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 82
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "项目"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 129
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "环境"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 66
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "类型"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 85
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "名称"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 165
              },
              {
                "id": "custom.filterable",
                "value": true
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "连通性"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 55
              },
              {
                "id": "mappings",
                "value": [
                  {
                    "options": {
                      "0": {
                        "color": "red",
                        "index": 1,
                        "text": "离线"
                      },
                      "1": {
                        "color": "green",
                        "index": 0,
                        "text": "在线"
                      }
                    },
                    "type": "value"
                  }
                ]
              },
              {
                "id": "custom.displayMode",
                "value": "color-background"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "状态"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 41
              },
              {
                "id": "decimals",
                "value": 0
              },
              {
                "id": "thresholds",
                "value": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "green",
                      "value": null
                    },
                    {
                      "color": "#EAB839",
                      "value": 400
                    },
                    {
                      "color": "red",
                      "value": 500
                    }
                  ]
                }
              },
              {
                "id": "custom.displayMode",
                "value": "color-background"
              },
              {
                "id": "mappings",
                "value": [
                  {
                    "options": {
                      "match": "null",
                      "result": {
                        "color": "red",
                        "index": 0,
                        "text": "无"
                      }
                    },
                    "type": "special"
                  }
                ]
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "证书(天)"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 70
              },
              {
                "id": "unit",
                "value": "none"
              },
              {
                "id": "thresholds",
                "value": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "red",
                      "value": null
                    },
                    {
                      "color": "#EAB839",
                      "value": 15
                    },
                    {
                      "color": "green",
                      "value": 30
                    }
                  ]
                }
              },
              {
                "id": "custom.displayMode",
                "value": "color-background"
              },
              {
                "id": "decimals",
                "value": 0
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "http版本"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 66
              },
              {
                "id": "custom.displayMode",
                "value": "color-background"
              },
              {
                "id": "decimals",
                "value": 1
              },
              {
                "id": "thresholds",
                "value": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "red",
                      "value": null
                    },
                    {
                      "color": "yellow",
                      "value": 1.1
                    },
                    {
                      "color": "green",
                      "value": 2
                    }
                  ]
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "耗时"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 123
              },
              {
                "id": "unit",
                "value": "s"
              },
              {
                "id": "custom.displayMode",
                "value": "gradient-gauge"
              },
              {
                "id": "decimals",
                "value": 1
              },
              {
                "id": "max",
                "value": 0.5
              },
              {
                "id": "color",
                "value": {
                  "mode": "continuous-GrYlRd"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "1H可用%"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 103
              },
              {
                "id": "unit",
                "value": "percentunit"
              },
              {
                "id": "decimals"
              },
              {
                "id": "thresholds",
                "value": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "red",
                      "value": null
                    },
                    {
                      "color": "yellow",
                      "value": 0.9
                    },
                    {
                      "color": "green",
                      "value": 0.99
                    }
                  ]
                }
              },
              {
                "id": "custom.displayMode",
                "value": "basic"
              },
              {
                "id": "max",
                "value": 1
              },
              {
                "id": "color",
                "value": {
                  "mode": "thresholds"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "实例"
            },
            "properties": [
              {
                "id": "custom.width"
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/连通性|状态|证书\\(天\\)|http版本|耗时|1H可用%|TLS版本|DNS/"
            },
            "properties": [
              {
                "id": "custom.align",
                "value": "center"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "实例"
            },
            "properties": [
              {
                "id": "links",
                "value": [
                  {
                    "targetBlank": true,
                    "title": "${__data.fields.名称}",
                    "url": "${__data.fields.实例}"
                  }
                ]
              },
              {
                "id": "custom.filterable",
                "value": true
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "TLS版本"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 63
              },
              {
                "id": "mappings",
                "value": [
                  {
                    "options": {
                      "TLS 1.0": {
                        "color": "red",
                        "index": 0,
                        "text": "1.0"
                      },
                      "TLS 1.1": {
                        "color": "yellow",
                        "index": 1,
                        "text": "1.1"
                      },
                      "TLS 1.2": {
                        "color": "green",
                        "index": 2,
                        "text": "1.2"
                      },
                      "TLS 1.3": {
                        "color": "green",
                        "index": 3,
                        "text": "1.3"
                      }
                    },
                    "type": "value"
                  }
                ]
              },
              {
                "id": "custom.displayMode",
                "value": "color-background"
              },
              {
                "id": "thresholds",
                "value": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "red",
                      "value": null
                    }
                  ]
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "DNS"
            },
            "properties": [
              {
                "id": "unit",
                "value": "s"
              },
              {
                "id": "custom.width",
                "value": 68
              },
              {
                "id": "custom.displayMode",
                "value": "color-background"
              },
              {
                "id": "decimals"
              },
              {
                "id": "thresholds",
                "value": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "green",
                      "value": null
                    },
                    {
                      "color": "#EAB839",
                      "value": 0.01
                    },
                    {
                      "color": "red",
                      "value": 0.02
                    }
                  ]
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "重定向"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 60
              },
              {
                "id": "decimals",
                "value": 0
              },
              {
                "id": "mappings",
                "value": [
                  {
                    "options": {
                      "0": {
                        "color": "blue",
                        "index": 0,
                        "text": "无"
                      },
                      "1": {
                        "index": 1,
                        "text": "1次"
                      },
                      "2": {
                        "index": 2,
                        "text": "2次"
                      },
                      "3": {
                        "index": 3,
                        "text": "3次"
                      },
                      "4": {
                        "index": 4,
                        "text": "4次"
                      },
                      "5": {
                        "index": 5,
                        "text": "5次"
                      }
                    },
                    "type": "value"
                  }
                ]
              },
              {
                "id": "custom.displayMode",
                "value": "color-background"
              },
              {
                "id": "thresholds",
                "value": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "orange",
                      "value": null
                    }
                  ]
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 5,
        "w": 24,
        "x": 0,
        "y": 1
      },
      "id": 2414,
      "links": [],
      "options": {
        "footer": {
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": true,
        "sortBy": [
          {
            "desc": false,
            "displayName": "类型"
          }
        ]
      },
      "pluginVersion": "7.5.11",
      "repeatDirection": "h",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "Prometheus"
          },
          "exemplar": false,
          "expr": "probe_success{module=~\"$module\",service=~\"$service\",instance=~\"$instance\",instance=~\". *$sname.*\"} - 0",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "Prometheus"
          },
          "exemplar": false,
          "expr": "probe_http_status_code{module=~\"$module\",service=~\"$service\",instance=~\"$instance\",instance=~\". *$sname.*\"} - 0",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "",
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "Prometheus"
          },
          "exemplar": false,
          "expr": "(probe_ssl_earliest_cert_expiry{module=~\"$module\",company=~\"$company\",project=~\"$project\",env=~\"$env\",name=~\"$name\",instance=~\"$instance\",instance=~\". *$sname.*\"}-time())/3600/24",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "",
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "Prometheus"
          },
          "exemplar": false,
          "expr": "probe_http_version{module=~\"$module\",company=~\"$company\",project=~\"$project\",env=~\"$env\",name=~\"$name\",instance=~\"$instance\",instance=~\". *$sname.*\"} - 0",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "Prometheus"
          },
          "exemplar": false,
          "expr": "probe_duration_seconds{module=~\"$module\",company=~\"$company\",project=~\"$project\",env=~\"$env\",name=~\"$name\",instance=~\"$instance\",instance=~\". *$sname.*\"} - 0",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "Prometheus"
          },
          "exemplar": false,
          "expr": "sum_over_time(probe_success{module=~\"$module\",company=~\"$company\",project=~\"$project\",env=~\"$env\",name=~\"$name\",instance=~\"$instance\",instance=~\". *$sname.*\"}[60m])/count_over_time(probe_success{module=~\"$module\",company=~\"$company\",project=~\"$project\",env=~\"$env\",name=~\"$name\",instance=~\"$instance\",instance=~\". *$sname.*\"}[60m])",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "Prometheus"
          },
          "exemplar": false,
          "expr": "probe_tls_version_info{module=~\"$module\",company=~\"$company\",project=~\"$project\",env=~\"$env\",name=~\"$name\",instance=~\"$instance\",instance=~\". *$sname.*\"}",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "Prometheus"
          },
          "exemplar": false,
          "expr": "probe_dns_lookup_time_seconds{module=~\"$module\",company=~\"$company\",project=~\"$project\",env=~\"$env\",name=~\"$name\",instance=~\"$instance\",instance=~\". *$sname.*\"}-0",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "H"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "Prometheus"
          },
          "exemplar": false,
          "expr": "probe_http_redirects{module=~\"$module\",company=~\"$company\",project=~\"$project\",env=~\"$env\",name=~\"$name\",instance=~\"$instance\",instance=~\". *$sname.*\"}-0",
          "format": "table",
          "hide": false,
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "I"
        }
      ],
      "title": "状态明细,资源数量:$count",
      "transformations": [
        {
          "id": "merge",
          "options": {}
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true,
              "Value #A": false,
              "Value #B": true,
              "__name __": true,
              "endpoint": true,
              "job": true,
              "origin_prometheus": true
            },
            "indexByName": {
              "Time": 0,
              "Value #A": 4,
              "Value #C": 5,
              "container": 9,
              "endpoint": 10,
              "instance": 3,
              "job": 2,
              "module": 1,
              "namespace": 6,
              "pod": 8,
              "service": 7
            },
            "renameByName": {
              "Value #A": "连通性",
              "Value #B": "",
              "Value #C": "状态",
              "Value #D": "证书(天)",
              "Value #E": "http版本",
              "Value #F": "耗时",
              "Value #G": "1H可用%",
              "Value #H": "DNS",
              "Value #I": "重定向",
              "company": "公司",
              "env": "环境",
              "instance": "实例",
              "job": "",
              "module": "类型",
              "name": "名称",
              "project": "项目",
              "version": "TLS版本"
            }
          }
        },
        {
          "id": "filterByValue",
          "options": {
            "filters": [
              {
                "config": {
                  "id": "isNull",
                  "options": {}
                },
                "fieldName": "连通性"
              },
              {
                "config": {
                  "id": "isNull",
                  "options": {}
                },
                "fieldName": "状态"
              }
            ],
            "match": "all",
            "type": "exclude"
          }
        }
      ],
      "type": "table"
    },
    {
      "collapsed": false,
      "datasource": null,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 6
      },
      "id": 2429,
      "panels": [],
      "title": "明细",
      "type": "row"
    },
    {
      "datasource": null,
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "mappings": [
            {
              "options": {
                "0": {
                  "index": 1,
                  "text": "离线"
                },
                "1": {
                  "index": 0,
                  "text": "在线"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 4,
        "x": 0,
        "y": 7
      },
      "id": 2427,
      "links": [],
      "options": {
        "displayLabels": [
          "percent"
        ],
        "legend": {
          "calcs": [],
          "displayMode": "table",
          "placement": "bottom",
          "values": [
            "value",
            "percent"
          ]
        },
        "pieType": "pie",
        "reduceOptions": {
          "calcs": [
            "count"
          ],
          "fields": "",
          "values": true
        },
        "text": {},
        "tooltip": {
          "mode": "multi"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "Prometheus"
          },
          "exemplar": true,
          "expr": "count_values('value',probe_success{module=~\"$module\",service=~\"$service\",instance=~\"$instance\",instance=~\". *$sname.*\"})",
          "format": "time_series",
          "instant": true,
          "interval": "",
          "legendFormat": "{{value}}",
          "refId": "A"
        }
      ],
      "title": "连通性占比",
      "transformations": [],
      "type": "piechart"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": null,
      "decimals": 0,
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 9,
        "w": 9,
        "x": 4,
        "y": 7
      },
      "hiddenSeries": false,
      "id": 24,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "hideEmpty": false,
        "hideZero": false,
        "max": true,
        "min": false,
        "rightSide": false,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "maxDataPoints": 100,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": false
      },
      "percentage": false,
      "pluginVersion": "7.5.11",
      "pointradius": 1,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "Prometheus"
          },
          "exemplar": true,
          "expr": "probe_duration_seconds{module=~\"$module\",service=~\"$service\",instance=~\"$instance\",instance=~\". *$sname.*\"}",
          "format": "time_series",
          "instant": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{service}}:{{instance}}",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "检测总耗时",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:443",
          "format": "s",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:444",
          "format": "none",
          "logBase": 1,
          "show": false
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": null,
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 16,
        "w": 11,
        "x": 13,
        "y": 7
      },
      "hiddenSeries": false,
      "id": 2420,
      "legend": {
        "alignAsTable": true,
        "avg": true,
        "current": true,
        "hideEmpty": true,
        "hideZero": true,
        "max": true,
        "min": true,
        "rightSide": false,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": true,
      "linewidth": 1,
      "links": [],
      "maxDataPoints": 100,
      "maxPerRow": 3,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.5.11",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
      "repeatDirection": "h",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "Prometheus"
          },
          "exemplar": true,
          "expr": "probe_http_duration_seconds{module=~\"$module\",service=~\"$service\",instance=~\"$instance\",instance=~\". *$sname.*\"}",
          "format": "time_series",
          "hide": false,
          "instant": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{service}}:{{instance}}",
          "refId": "B"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "ICMP/HTTPS检测类 阶段耗时",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:532",
          "format": "s",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:533",
          "format": "short",
          "logBase": 1,
          "show": false
        }
      ],
      "yaxis": {
        "align": false
      }
    },
    {
      "datasource": null,
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 4,
        "x": 0,
        "y": 15
      },
      "id": 2426,
      "links": [],
      "options": {
        "displayLabels": [],
        "legend": {
          "calcs": [],
          "displayMode": "table",
          "placement": "bottom",
          "sortBy": "Percent",
          "sortDesc": true,
          "values": [
            "value",
            "percent"
          ]
        },
        "pieType": "pie",
        "reduceOptions": {
          "calcs": [
            "count"
          ],
          "fields": "",
          "values": true
        },
        "text": {},
        "tooltip": {
          "mode": "multi"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "Prometheus"
          },
          "exemplar": false,
          "expr": "count_values('value',probe_http_status_code{module=~\"$module\",service=~\"$service\",instance=~\"$instance\",instance=~\". *$sname.*\"})",
          "format": "time_series",
          "instant": true,
          "interval": "",
          "legendFormat": "{{value}}",
          "refId": "A"
        }
      ],
      "title": "HTTP 状态占比",
      "transformations": [],
      "type": "piechart"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": null,
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 7,
        "w": 9,
        "x": 4,
        "y": 16
      },
      "hiddenSeries": false,
      "id": 28,
      "legend": {
        "alignAsTable": true,
        "avg": false,
        "current": true,
        "max": true,
        "min": true,
        "rightSide": false,
        "show": true,
        "sort": "current",
        "sortDesc": true,
        "total": false,
        "values": true
      },
      "lines": false,
      "linewidth": 1,
      "links": [],
      "maxDataPoints": 100,
      "maxPerRow": 3,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.5.11",
      "pointradius": 1,
      "points": true,
      "renderer": "flot",
      "repeatDirection": "h",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "Prometheus"
          },
          "exemplar": true,
          "expr": "probe_http_status_code{module=~\"$module\",service=~\"$service\",instance=~\"$instance\",instance=~\". *$sname.*\"}",
          "format": "time_series",
          "hide": false,
          "interval": "",
          "intervalFactor": 1,
          "legendFormat": "{{service}}:{{instacance}} ",
          "refId": "B"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "网站HTTP状态检查",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "$$hashKey": "object:487",
          "format": "short",
          "logBase": 1,
          "show": true
        },
        {
          "$$hashKey": "object:488",
          "format": "short",
          "logBase": 1,
          "show": false
        }
      ],
      "yaxis": {
        "align": false
      }
    }
  ],
  "refresh": "",
  "schemaVersion": 27,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "allValue": null,
        "current": {
          "selected": false,
          "text": "All",
          "value": "$__all"
        },
        "datasource": null,
        "definition": "label_values(probe_success,module)",
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": true,
        "label": "类型",
        "multi": false,
        "name": "module",
        "options": [],
        "query": {
          "query": "label_values(probe_success,module)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 5,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": ". *",
        "current": {
          "selected": false,
          "text": "All",
          "value": "$__all"
        },
        "datasource": null,
        "definition": "label_values(probe_success{module=~\"$module\"},service)",
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": true,
        "label": "服务",
        "multi": false,
        "name": "service",
        "options": [],
        "query": {
          "query": "label_values(probe_success{module=~\"$module\"},service)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 5,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {
          "selected": false,
          "text": "All",
          "value": "$__all"
        },
        "datasource": null,
        "definition": "label_values(probe_success{module=~\"$module\",service=~\"$service\"},instance)",
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": true,
        "label": "实例",
        "multi": false,
        "name": "instance",
        "options": [],
        "query": {
          "query": "label_values(probe_success{module=~\"$module\",service=~\"$service\"},instance)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {
          "selected": false,
          "text": "3",
          "value": "3"
        },
        "datasource": "Prometheus",
        "definition": "query_result(count(probe_success{module=~\"$module\",service=~\"$service\",instance=~\"$instance\",instance=~\".*$sname. *\"}))",
        "description": null,
        "error": null,
        "hide": 2,
        "includeAll": false,
        "label": "",
        "multi": false,
        "name": "count",
        "options": [],
        "query": {
          "query": "query_result(count(probe_success{module=~\"$module\",service=~\"$service\",instance=~\"$instance\",instance=~\".*$sname. *\"}))",
          "refId": "StandardVariableQuery"
        },
        "refresh": 1,
        "regex": "/{} (.*) . */",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "current": {
          "selected": false,
          "text": "",
          "value": ""
        },
        "description": null,
        "error": null,
        "hide": 0,
        "label": "URL筛选(正则)",
        "name": "sname",
        "options": [
          {
            "selected": true,
            "text": "",
            "value": ""
          }
        ],
        "query": "",
        "skipUrlSync": false,
        "type": "textbox"
      }
    ]
  },
  "time": {
    "from": "now-15m",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "browser",
  "title": "Blackbox Exporter Dashboard",
  "uid": "Sj56fJUnz",
  "version": 2
}


四.注意事项

根据实际情况对配置进行相关修改

无.结果检查

手动修改service相关配置,查看grafana仪表盘上的监控数据是否符合预期


相关文章

kafka常见配置参数解析

broker.idbroker 的全局唯一编号,不能重复,只能是数字num.network.threads=3处理网络请求的线程数量num.io.threads=8用来处理磁盘 IO 的线程数量soc...

MySQL日期相关函数

一、当前时间--当前时间select current_timestamp();--当前时间 等价于 select current_timestamp();select now();--基本等价于以上两...

Flink window详解

Flink window详解

一、窗口(window)一般真实的流都是无界的,如果是无界怎样处理无界的数据可以把无限的数据流进行切分,得到有限的数据集进行处理 —— 也 就是得到有界流 窗口(window)就是将无限流切割为有限流...

dbms_support包跟踪10046

系统默认没有安装dbms_support这个包,可以手动执行$ORACLE_HOME/rdbms/admin/dbmssupp.sql脚本来创建该包。安装dbms_support包: SQL>...

Elasticsearch如何使用内存

ES作为一个JAVA程序,其对内存的使用和管理依赖底层JVM。因而设置内存时需要遵从JAVA的普适原则,如-xmx和-xms设置为相同值等。在JVM的基础上,ES对内存的使用可按功能分为以下几大部分:...

HDP-Yarn开启CPU调度和隔离

HDP-Yarn开启CPU调度和隔离

进入到ambari主界面 点击yarn 点击config CPU Scheduling and Isolation 设置为enable修改高级配置点击ADVANCED搜索需要修改的配yarn.node...

发表评论    

◎欢迎参与讨论,请在这里发表您的看法、交流您的观点。