Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Prometheus should scrape the router by default #18254

Merged
merged 2 commits into from
Mar 1, 2018
Merged

Prometheus should scrape the router by default #18254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a63df7b
Router should expose metrics via service serving cert
smarterclayton Jan 23, 2018
0786dad
Prometheus should scrape the router by default
smarterclayton Jan 23, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 85 additions & 3 deletions examples/prometheus/prometheus.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,53 @@ objects:
serviceaccounts.openshift.io/oauth-redirectreference.prom: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"prometheus"}}'
serviceaccounts.openshift.io/oauth-redirectreference.alerts: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"alerts"}}'
serviceaccounts.openshift.io/oauth-redirectreference.alertmanager: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"alertmanager"}}'

# Create a service account for accessing prometheus data
- apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus-reader
namespace: "${NAMESPACE}"

# Create a service account for prometheus to use to scrape other infrastructure components
- apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus-scraper
namespace: "${NAMESPACE}"

- apiVersion: v1
kind: Secret
metadata:
name: prometheus-scraper
namespace: "${NAMESPACE}"
annotations:
kubernetes.io/service-account.name: prometheus-scraper
type: kubernetes.io/service-account-token

- apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: prometheus-scraper
rules:
- apiGroups:
- route.openshift.io
resources:
- routers/metrics
verbs:
- get

- apiVersion: authorization.openshift.io/v1
kind: ClusterRoleBinding
metadata:
name: prometheus-scraper
roleRef:
name: prometheus-scraper
subjects:
- kind: ServiceAccount
name: prometheus-scraper
namespace: "${NAMESPACE}"

- apiVersion: authorization.openshift.io/v1
kind: ClusterRoleBinding
metadata:
Expand All @@ -51,6 +98,18 @@ objects:
name: prometheus
namespace: "${NAMESPACE}"

- apiVersion: authorization.openshift.io/v1
kind: RoleBinding
metadata:
name: prometheus-reader
namespace: "${NAMESPACE}"
roleRef:
name: view
subjects:
- kind: ServiceAccount
name: prometheus-reader
namespace: "${NAMESPACE}"

# Create a fully end-to-end TLS connection to the prometheus proxy
- apiVersion: route.openshift.io/v1
kind: Route
Expand Down Expand Up @@ -229,6 +288,8 @@ objects:
name: prometheus-config
- mountPath: /prometheus
name: prometheus-data
- mountPath: /var/run/secrets/kubernetes.io/scraper
name: prometheus-scraper-secret

# Deploy alertmanager behind prometheus-alert-buffer behind an oauth proxy
# use http port=4190 and https port=9943 to differ from prom-proxy
Expand Down Expand Up @@ -320,6 +381,9 @@ objects:
configMap:
defaultMode: 420
name: prometheus
- name: prometheus-scraper-secret
secret:
secretName: prometheus-scraper
- name: prometheus-proxy-secret
secret:
secretName: prometheus-proxy
Expand Down Expand Up @@ -603,21 +667,39 @@ objects:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)

# TODO: auto-generate these sections, or add a dynamic infrastructure scraper
# Scrape config for the template service broker
- job_name: 'openshift-template-service-broker'
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
server_name: apiserver.openshift-template-service-broker.svc
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

bearer_token_file: /var/run/secrets/kubernetes.io/scraper/token
kubernetes_sd_configs:
- role: endpoints

namespaces:
names:
- openshift-template-service-broker
relabel_configs:
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
action: keep
regex: openshift-template-service-broker;apiserver;https
# Scrape config for the router
- job_name: 'openshift-router'
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
server_name: router.default.svc
bearer_token_file: /var/run/secrets/kubernetes.io/scraper/token
kubernetes_sd_configs:
- role: endpoints
namespaces:
names:
- default
relabel_configs:
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
action: keep
regex: default;router;1936-tcp

alerting:
alertmanagers:
Expand Down
63 changes: 26 additions & 37 deletions pkg/oc/admin/router/router.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,27 @@ func generateSecretsConfig(cfg *RouterConfig, namespace string, defaultCert []by
secrets = append(secrets, secret)
}

if cfg.Type == "haproxy-router" && cfg.StatsPort != 0 {
metricsCertName := "router-metrics-tls"
if len(defaultCert) == 0 {
// when we are generating a serving cert, we need to reuse the existing cert
metricsCertName = certName
}
volumes = append(volumes, kapi.Volume{
Name: "metrics-server-certificate",
VolumeSource: kapi.VolumeSource{
Secret: &kapi.SecretVolumeSource{
SecretName: metricsCertName,
},
},
})
mounts = append(mounts, kapi.VolumeMount{
Name: "metrics-server-certificate",
ReadOnly: true,
MountPath: "/etc/pki/tls/metrics/",
})
}

// The secret in this volume is either the one created for the
// user supplied default cert (pem format) or the secret generated
// by the service anotation (cert only format).
Expand Down Expand Up @@ -460,36 +481,6 @@ func generateReadinessProbeConfig(cfg *RouterConfig, ports []kapi.ContainerPort)
return probe
}

func generateMetricsExporterContainer(cfg *RouterConfig, env app.Environment) *kapi.Container {
containerName := "metrics-exporter"
if len(cfg.MetricsImage) > 0 {
return &kapi.Container{
Name: containerName,
Image: cfg.MetricsImage,
Env: env.List(),
}
}
switch cfg.Type {
case "haproxy-router":
return &kapi.Container{
Name: containerName,
Image: "prom/haproxy-exporter:latest",
Env: env.List(),
Args: []string{
fmt.Sprintf("--haproxy.scrape-uri=http://$(STATS_USERNAME):$(STATS_PASSWORD)@localhost:$(STATS_PORT)/haproxy?stats;csv"),
},
Ports: []kapi.ContainerPort{
{
Name: "http",
ContainerPort: 9101,
},
},
}
default:
return nil
}
}

// RunCmdRouter contains all the necessary functionality for the
// OpenShift CLI router command.
func RunCmdRouter(f *clientcmd.Factory, cmd *cobra.Command, out, errout io.Writer, cfg *RouterConfig, args []string) error {
Expand Down Expand Up @@ -696,6 +687,8 @@ func RunCmdRouter(f *clientcmd.Factory, cmd *cobra.Command, out, errout io.Write
if cfg.Type == "haproxy-router" && cfg.StatsPort != 0 {
env["ROUTER_LISTEN_ADDR"] = fmt.Sprintf("0.0.0.0:%d", cfg.StatsPort)
env["ROUTER_METRICS_TYPE"] = "haproxy"
env["ROUTER_METRICS_TLS_CERT_FILE"] = "/etc/pki/tls/metrics/tls.crt"
env["ROUTER_METRICS_TLS_KEY_FILE"] = "/etc/pki/tls/metrics/tls.key"
}
env.Add(secretEnv)
if len(defaultCert) > 0 {
Expand Down Expand Up @@ -741,13 +734,6 @@ func RunCmdRouter(f *clientcmd.Factory, cmd *cobra.Command, out, errout io.Write
},
}

if cfg.StatsPort > 0 && cfg.ExposeMetrics {
pc := generateMetricsExporterContainer(cfg, env)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Think the generateMetricsExporterContainer function isn't used anywhere else, if you are removing this code.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed

if pc != nil {
containers = append(containers, *pc)
}
}

objects := []runtime.Object{}
for _, s := range secrets {
objects = append(objects, s)
Expand Down Expand Up @@ -826,6 +812,9 @@ func RunCmdRouter(f *clientcmd.Factory, cmd *cobra.Command, out, errout io.Write
// The secret generated by the service annotaion contains a tls.crt and tls.key
// which ultimately need to be combined into a pem
t.Annotations["service.alpha.openshift.io/serving-cert-secret-name"] = certName
} else if cfg.Type == "haproxy-router" && cfg.StatsPort != 0 {
// Generate a serving cert for metrics only
t.Annotations["service.alpha.openshift.io/serving-cert-secret-name"] = "router-metrics-tls"
}
}
}
Expand Down
88 changes: 85 additions & 3 deletions pkg/oc/bootstrap/bindata.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading