prometheus.yaml

apiVersion: template.openshift.io/v1
kind: Template
metadata:
  name: prometheus
  annotations:
    "openshift.io/display-name": Prometheus
    description: |
      A monitoring solution for an OpenShift cluster - collect and gather metrics and alerts from nodes, services, and the infrastructure. This is a tech preview feature.
    iconClass: fa fa-cogs
    tags: "monitoring,prometheus, alertmanager,time-series"
parameters:
- description: The namespace to instantiate prometheus under. Defaults to 'kube-system'.
  name: NAMESPACE
  value: kube-system
- description: The location of the proxy image
  name: IMAGE_PROXY
  value: openshift/oauth-proxy:v1.0.0
- description: The location of the prometheus image
  name: IMAGE_PROMETHEUS
  value: openshift/prometheus:v2.0.0
- description: The location of the alertmanager image
  name: IMAGE_ALERTMANAGER
  value: openshift/prometheus-alertmanager:v0.13.0
- description: The location of alert-buffer image
  name: IMAGE_ALERT_BUFFER
  value: openshift/prometheus-alert-buffer:v0.0.2
- description: The session secret for the proxy
  name: SESSION_SECRET
  generate: expression
  from: "[a-zA-Z0-9]{43}"

objects:
# Authorize the prometheus service account to read data about the cluster
- apiVersion: v1
  kind: ServiceAccount
  metadata:
    name: prometheus
    namespace: "${NAMESPACE}"
    annotations:
      serviceaccounts.openshift.io/oauth-redirectreference.prom: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"prometheus"}}'
      serviceaccounts.openshift.io/oauth-redirectreference.alerts: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"alerts"}}'
      serviceaccounts.openshift.io/oauth-redirectreference.alertmanager: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"alertmanager"}}'

# Create a service account for accessing prometheus data
- apiVersion: v1
  kind: ServiceAccount
  metadata:
    name: prometheus-reader
    namespace: "${NAMESPACE}"

# Create a service account for prometheus to use to scrape other infrastructure components
- apiVersion: v1
  kind: ServiceAccount
  metadata:
    name: prometheus-scraper
    namespace: "${NAMESPACE}"

- apiVersion: v1
  kind: Secret
  metadata:
    name: prometheus-scraper
    namespace: "${NAMESPACE}"
    annotations:
      kubernetes.io/service-account.name: prometheus-scraper
  type: kubernetes.io/service-account-token

- apiVersion: rbac.authorization.k8s.io/v1
  kind: ClusterRole
  metadata:
    name: prometheus-scraper
  rules:
  - apiGroups:
    - route.openshift.io
    resources:
    - routers/metrics
    verbs:
    - get

- apiVersion: authorization.openshift.io/v1
  kind: ClusterRoleBinding
  metadata:
    name: prometheus-scraper
  roleRef:
    name: prometheus-scraper
  subjects:
  - kind: ServiceAccount
    name: prometheus-scraper
    namespace: "${NAMESPACE}"

- apiVersion: authorization.openshift.io/v1
  kind: ClusterRoleBinding
  metadata:
    name: prometheus-cluster-reader
  roleRef:
    name: cluster-reader
  subjects:
  - kind: ServiceAccount
    name: prometheus
    namespace: "${NAMESPACE}"

- apiVersion: authorization.openshift.io/v1
  kind: RoleBinding
  metadata:
    name: prometheus-reader
    namespace: "${NAMESPACE}"
  roleRef:
    name: view
  subjects:
  - kind: ServiceAccount
    name: prometheus-reader
    namespace: "${NAMESPACE}"

# Create a fully end-to-end TLS connection to the prometheus proxy
- apiVersion: route.openshift.io/v1
  kind: Route
  metadata:
    name: prometheus
    namespace: "${NAMESPACE}"
  spec:
    to:
      name: prometheus
    tls:
      termination: Reencrypt
      insecureEdgeTerminationPolicy: Redirect
- apiVersion: v1
  kind: Service
  metadata:
    annotations:
      prometheus.io/scrape: "true"
      prometheus.io/scheme: https
      service.alpha.openshift.io/serving-cert-secret-name: prometheus-tls
    labels:
      name: prometheus
    name: prometheus
    namespace: "${NAMESPACE}"
  spec:
    ports:
    - name: prometheus
      port: 443
      protocol: TCP
      targetPort: 8443
    selector:
      app: prometheus
- apiVersion: v1
  kind: Secret
  metadata:
    name: prometheus-proxy
    namespace: "${NAMESPACE}"
  stringData:
    session_secret: "${SESSION_SECRET}="

# Create a fully end-to-end TLS connection to the alert proxy
- apiVersion: route.openshift.io/v1
  kind: Route
  metadata:
    name: alerts
    namespace: "${NAMESPACE}"
  spec:
    to:
      name: alerts
    tls:
      termination: Reencrypt
      insecureEdgeTerminationPolicy: Redirect
- apiVersion: v1
  kind: Service
  metadata:
    annotations:
      service.alpha.openshift.io/serving-cert-secret-name: alerts-tls
    labels:
      name: alerts
    name: alerts
    namespace: "${NAMESPACE}"
  spec:
    ports:
    - name: alerts
      port: 443
      protocol: TCP
      targetPort: 9443
    selector:
      app: prometheus
- apiVersion: v1
  kind: Secret
  metadata:
    name: alerts-proxy
    namespace: "${NAMESPACE}"
  stringData:
    session_secret: "${SESSION_SECRET}="

# Create a fully end-to-end TLS connection to the alertmanager proxy
- apiVersion: route.openshift.io/v1
  kind: Route
  metadata:
    name: alertmanager
    namespace: "${NAMESPACE}"
  spec:
    to:
      name: alertmanager
    tls:
      termination: Reencrypt
      insecureEdgeTerminationPolicy: Redirect
- apiVersion: v1
  kind: Service
  metadata:
    annotations:
      service.alpha.openshift.io/serving-cert-secret-name: alertmanager-tls
    labels:
      name: alertmanager
    name: alertmanager
    namespace: "${NAMESPACE}"
  spec:
    ports:
    - name: alertmanager
      port: 443
      protocol: TCP
      targetPort: 10443
    selector:
      app: prometheus
- apiVersion: v1
  kind: Secret
  metadata:
    name: alertmanager-proxy
    namespace: "${NAMESPACE}"
  stringData:
    session_secret: "${SESSION_SECRET}="

- apiVersion: apps/v1beta1
  kind: StatefulSet
  metadata:
    labels:
      app: prometheus
    name: prometheus
    namespace: "${NAMESPACE}"
  spec:
    updateStrategy:
      type: RollingUpdate
    podManagementPolicy: Parallel
    selector:
      matchLabels:
        app: prometheus
    template:
      metadata:
        labels:
          app: prometheus
        name: prometheus
      spec:
        serviceAccountName: prometheus
        containers:
        # Deploy Prometheus behind an oauth proxy
        - name: prom-proxy
          image: ${IMAGE_PROXY}
          imagePullPolicy: IfNotPresent
          ports:
          - containerPort: 8443
            name: web
          args:
          - -provider=openshift
          - -https-address=:8443
          - -http-address=
          - -email-domain=*
          - -upstream=http://localhost:9090
          - -client-id=system:serviceaccount:${NAMESPACE}:prometheus
          - -openshift-ca=/etc/pki/tls/cert.pem
          - -openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          - '-openshift-sar={"resource": "namespaces", "verb": "get", "resourceName": "${NAMESPACE}", "namespace": "${NAMESPACE}"}'
          - '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get", "resourceName": "${NAMESPACE}", "namespace": "${NAMESPACE}"}}'
          - -tls-cert=/etc/tls/private/tls.crt
          - -tls-key=/etc/tls/private/tls.key
          - -client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token
          - -cookie-secret-file=/etc/proxy/secrets/session_secret
          - -skip-auth-regex=^/metrics
          volumeMounts:
          - mountPath: /etc/tls/private
            name: prometheus-tls-secret
          - mountPath: /etc/proxy/secrets
            name: prometheus-proxy-secret
          - mountPath: /prometheus
            name: prometheus-data

        - name: prometheus
          args:
          - --storage.tsdb.retention=6h
          - --config.file=/etc/prometheus/prometheus.yml
          - --web.listen-address=localhost:9090
          image: ${IMAGE_PROMETHEUS}
          imagePullPolicy: IfNotPresent
          volumeMounts:
          - mountPath: /etc/prometheus
            name: prometheus-config
          - mountPath: /prometheus
            name: prometheus-data
          - mountPath: /var/run/secrets/kubernetes.io/scraper
            name: prometheus-scraper-secret

        # Deploy alertmanager behind prometheus-alert-buffer behind an oauth proxy
        # use http port=4190 and https port=9943 to differ from prom-proxy
        - name: alerts-proxy
          image: ${IMAGE_PROXY}
          imagePullPolicy: IfNotPresent
          ports:
          - containerPort: 9443
            name: web
          args:
          - -provider=openshift
          - -https-address=:9443
          - -http-address=
          - -email-domain=*
          - -upstream=http://localhost:9099
          - -client-id=system:serviceaccount:${NAMESPACE}:prometheus
          - -openshift-ca=/etc/pki/tls/cert.pem
          - -openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          - '-openshift-sar={"resource": "namespaces", "verb": "get", "resourceName": "${NAMESPACE}", "namespace": "${NAMESPACE}"}'
          - '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get", "resourceName": "${NAMESPACE}", "namespace": "${NAMESPACE}"}}'
          - -tls-cert=/etc/tls/private/tls.crt
          - -tls-key=/etc/tls/private/tls.key
          - -client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token
          - -cookie-secret-file=/etc/proxy/secrets/session_secret
          volumeMounts:
          - mountPath: /etc/tls/private
            name: alerts-tls-secret
          - mountPath: /etc/proxy/secrets
            name: alerts-proxy-secrets

        - name: alert-buffer
          args:
          - --storage-path=/alert-buffer/messages.db
          image: ${IMAGE_ALERT_BUFFER}
          imagePullPolicy: IfNotPresent
          volumeMounts:
          - mountPath: /alert-buffer
            name: alerts-data
          ports:
          - containerPort: 9099
            name: alert-buf

        - name: alertmanager-proxy
          image: ${IMAGE_PROXY}
          imagePullPolicy: IfNotPresent
          ports:
          - containerPort: 10443
            name: web
          args:
          - -provider=openshift
          - -https-address=:10443
          - -http-address=
          - -email-domain=*
          - -upstream=http://localhost:9093
          - -client-id=system:serviceaccount:${NAMESPACE}:prometheus
          - -openshift-ca=/etc/pki/tls/cert.pem
          - -openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          - '-openshift-sar={"resource": "namespaces", "verb": "get", "resourceName": "${NAMESPACE}", "namespace": "${NAMESPACE}"}'
          - '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get", "resourceName": "${NAMESPACE}", "namespace": "${NAMESPACE}"}}'
          - -tls-cert=/etc/tls/private/tls.crt
          - -tls-key=/etc/tls/private/tls.key
          - -client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token
          - -cookie-secret-file=/etc/proxy/secrets/session_secret
          - -skip-auth-regex=^/metrics
          volumeMounts:
          - mountPath: /etc/tls/private
            name: alertmanager-tls-secret
          - mountPath: /etc/proxy/secrets
            name: alertmanager-proxy-secret

        - name: alertmanager
          args:
          - --config.file=/etc/alertmanager/alertmanager.yml
          image: ${IMAGE_ALERTMANAGER}
          imagePullPolicy: IfNotPresent
          ports:
          - containerPort: 9093
            name: web
          volumeMounts:
          - mountPath: /etc/alertmanager
            name: alertmanager-config
          - mountPath: /alertmanager
            name: alertmanager-data

        restartPolicy: Always
        volumes:

        - name: prometheus-config
          configMap:
            defaultMode: 420
            name: prometheus
        - name: prometheus-scraper-secret
          secret:
            secretName: prometheus-scraper
        - name: prometheus-proxy-secret
          secret:
            secretName: prometheus-proxy
        - name: prometheus-tls-secret
          secret:
            secretName: prometheus-tls
        - name: prometheus-data
          emptyDir: {}

        - name: alertmanager-config
          configMap:
            defaultMode: 420
            name: alertmanager
        - name: alertmanager-tls-secret
          secret:
            secretName: alertmanager-tls  
        - name: alertmanager-proxy-secret
          secret:
            secretName: alertmanager-proxy         

        - name: alerts-proxy-secrets
          secret:
            secretName: alerts-proxy
        - name: alerts-tls-secret
          secret:
            secretName: alerts-tls
        - name: alertmanager-data
          emptyDir: {}
        - name: alerts-data
          emptyDir: {}

- apiVersion: v1
  kind: ConfigMap
  metadata:
    name: prometheus
    namespace: "${NAMESPACE}"
  data:
    alerting.rules: |
      groups:
      - name: example-rules
        interval: 30s # defaults to global interval
        rules:
        - alert: Node Down
          expr: up{job="kubernetes-nodes"} == 0
          annotations:
            miqTarget: "ContainerNode"
            severity: "HIGH"
            message: "{{$labels.instance}} is down"
    
    recording.rules: |
      groups:
      - name: aggregate_container_resources
        rules:
        - record: container_cpu_usage_rate
          expr: sum without (cpu) (rate(container_cpu_usage_seconds_total[5m]))
        - record: container_memory_rss_by_type
          expr: container_memory_rss{id=~"/|/system.slice|/kubepods.slice"} > 0
        - record: container_cpu_usage_percent_by_host
          expr: sum by (hostname,type)(rate(container_cpu_usage_seconds_total{id="/"}[5m])) / on (hostname,type) machine_cpu_cores
        - record: apiserver_request_count_rate_by_resources
          expr: sum without (client,instance,contentType) (rate(apiserver_request_count[5m]))

    prometheus.yml: |
      rule_files:
        - '*.rules'

      # A scrape configuration for running Prometheus on a Kubernetes cluster.
      # This uses separate scrape configs for cluster components (i.e. API server, node)
      # and services to allow each to use different authentication configs.
      #
      # Kubernetes labels will be added as Prometheus labels on metrics via the
      # `labelmap` relabeling action.

      # Scrape config for API servers.
      #
      # Kubernetes exposes API servers as endpoints to the default/kubernetes
      # service so this uses `endpoints` role and uses relabelling to only keep
      # the endpoints associated with the default/kubernetes service using the
      # default named port `https`. This works for single API server deployments as
      # well as HA API server deployments.
      scrape_configs:
      - job_name: 'kubernetes-apiservers'

        kubernetes_sd_configs:
        - role: endpoints

        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

        # Keep only the default/kubernetes service endpoints for the https port. This
        # will add targets for each API server which Kubernetes adds an endpoint to
        # the default/kubernetes service.
        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
          action: keep
          regex: default;kubernetes;https

      # Scrape config for controllers.
      #
      # Each master node exposes a /metrics endpoint on :8444 that contains operational metrics for
      # the controllers.
      #
      # TODO: move this to a pure endpoints based metrics gatherer when controllers are exposed via
      #       endpoints.
      - job_name: 'kubernetes-controllers'

        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

        kubernetes_sd_configs:
        - role: endpoints

        # Keep only the default/kubernetes service endpoints for the https port, and then
        # set the port to 8444. This is the default configuration for the controllers on OpenShift
        # masters.
        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
          action: keep
          regex: default;kubernetes;https
        - source_labels: [__address__]
          action: replace
          target_label: __address__
          regex: (.+)(?::\d+)
          replacement: $1:8444

      # Scrape config for nodes.
      #
      # Each node exposes a /metrics endpoint that contains operational metrics for
      # the Kubelet and other components.
      - job_name: 'kubernetes-nodes'

        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

        kubernetes_sd_configs:
        - role: node

        # Drop a very high cardinality metric that is incorrect in 3.7. It will be
        # fixed in 3.9.
        metric_relabel_configs:
        - source_labels: [__name__]
          action: drop
          regex: 'openshift_sdn_pod_(setup|teardown)_latency(.*)'

        relabel_configs:
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)

      # Scrape config for cAdvisor.
      #
      # Beginning in Kube 1.7, each node exposes a /metrics/cadvisor endpoint that
      # reports container metrics for each running pod. Scrape those by default.
      - job_name: 'kubernetes-cadvisor'

        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

        metrics_path: /metrics/cadvisor

        kubernetes_sd_configs:
        - role: node

        # Exclude a set of high cardinality metrics that can contribute to significant
        # memory use in large clusters. These can be selectively enabled as necessary
        # for medium or small clusters.
        metric_relabel_configs:
        - source_labels: [__name__]
          action: drop
          regex: 'container_(cpu_user_seconds_total|cpu_cfs_periods_total|memory_usage_bytes|memory_swap|memory_working_set_bytes|memory_cache|last_seen|fs_(read_seconds_total|write_seconds_total|sector_(.*)|io_(.*)|reads_merged_total|writes_merged_total)|tasks_state|memory_failcnt|memory_failures_total|spec_memory_swap_limit_bytes|fs_(.*)_bytes_total|spec_(.*))'

        relabel_configs:
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)

      # Scrape config for service endpoints.
      #
      # The relabeling allows the actual service scrape endpoint to be configured
      # via the following annotations:
      #
      # * `prometheus.io/scrape`: Only scrape services that have a value of `true`
      # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need
      # to set this to `https` & most likely set the `tls_config` of the scrape config.
      # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
      # * `prometheus.io/port`: If the metrics are exposed on a different port to the
      # service then set this appropriately.
      - job_name: 'kubernetes-service-endpoints'

        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          # TODO: this should be per target
          insecure_skip_verify: true

        kubernetes_sd_configs:
        - role: endpoints

        relabel_configs:
        # only scrape infrastructure components
        - source_labels: [__meta_kubernetes_namespace]
          action: keep
          regex: 'default|logging|metrics|kube-.+|openshift|openshift-.+'
        # drop infrastructure components managed by other scrape targets
        - source_labels: [__meta_kubernetes_service_name]
          action: drop
          regex: 'prometheus-node-exporter'
        # only those that have requested scraping
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
          action: keep
          regex: true
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
          action: replace
          target_label: __scheme__
          regex: (https?)
        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
          action: replace
          target_label: __metrics_path__
          regex: (.+)
        - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
          action: replace
          target_label: __address__
          regex: (.+)(?::\d+);(\d+)
          replacement: $1:$2
        - action: labelmap
          regex: __meta_kubernetes_service_label_(.+)
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: kubernetes_namespace
        - source_labels: [__meta_kubernetes_service_name]
          action: replace
          target_label: kubernetes_name

      # Scrape config for node-exporter, which is expected to be running on port 9100.
      - job_name: 'kubernetes-nodes-exporter'

        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

        kubernetes_sd_configs:
        - role: node

        metric_relabel_configs:
        - source_labels: [__name__]
          action: drop
          regex: 'node_cpu|node_(disk|scrape_collector)_.+'
        # preserve a subset of the network, netstat, vmstat, and filesystem series
        - source_labels: [__name__]
          action: replace
          regex: '(node_(netstat_Ip_.+|vmstat_(nr|thp)_.+|filesystem_(free|size|device_error)|network_(transmit|receive)_(drop|errs)))'
          target_label: __name__
          replacement: renamed_$1
        - source_labels: [__name__]
          action: drop
          regex: 'node_(netstat|vmstat|filesystem|network)_.+'
        - source_labels: [__name__]
          action: replace
          regex: 'renamed_(.+)'
          target_label: __name__
          replacement: $1
        # drop any partial expensive series
        - source_labels: [__name__, device]
          action: drop
          regex: 'node_network_.+;veth.+'
        - source_labels: [__name__, mountpoint]
          action: drop
          regex: 'node_filesystem_(free|size|device_error);([^/].*|/.+)'

        relabel_configs:
        - source_labels: [__address__]
          regex: '(.*):10250'
          replacement: '${1}:9100'
          target_label: __address__
        - source_labels: [__meta_kubernetes_node_label_kubernetes_io_hostname]
          target_label: __instance__
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)

      # TODO: auto-generate these sections, or add a dynamic infrastructure scraper
      # Scrape config for the template service broker
      - job_name: 'openshift-template-service-broker'
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
          server_name: apiserver.openshift-template-service-broker.svc
        bearer_token_file: /var/run/secrets/kubernetes.io/scraper/token
        kubernetes_sd_configs:
        - role: endpoints
          namespaces:
            names:
            - openshift-template-service-broker
        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
          action: keep
          regex: openshift-template-service-broker;apiserver;https
      # Scrape config for the router
      - job_name: 'openshift-router'
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
          server_name: router.default.svc
        bearer_token_file: /var/run/secrets/kubernetes.io/scraper/token
        kubernetes_sd_configs:
        - role: endpoints
          namespaces:
            names:
            - default
        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
          action: keep
          regex: default;router;1936-tcp

      alerting:
        alertmanagers:
        - scheme: http
          static_configs:
          - targets:
            - "localhost:9093"

- apiVersion: v1
  kind: ConfigMap
  metadata:
    name: alertmanager
    namespace: "${NAMESPACE}"
  data:
    alertmanager.yml: |
      global:

      # The root route on which each incoming alert enters.
      route:
        # default route if none match
        receiver: alert-buffer-wh

        # The labels by which incoming alerts are grouped together. For example,
        # multiple alerts coming in for cluster=A and alertname=LatencyHigh would
        # be batched into a single group.
        # TODO:
        group_by: []

        # All the above attributes are inherited by all child routes and can
        # overwritten on each.

      receivers:
      - name: alert-buffer-wh
        webhook_configs:
        - url: http://localhost:9099/topics/alerts