From 5c120691dc90f438270a7fcdeaedd62288ed75c9 Mon Sep 17 00:00:00 2001 From: Jacob Tanenbaum Date: Fri, 11 May 2018 11:34:20 -0400 Subject: [PATCH] Differentiate liveness and readiness probes for router Add a backend to the router controller "/livez" that always returns true. This differentiates the liveness and readiness probes so that a router can be alive and not ready. Bug 1550007 --- pkg/cmd/infra/router/template.go | 15 ++++++++++++--- pkg/oc/admin/router/router.go | 19 +++++++++---------- pkg/router/metrics/health.go | 23 +++++++++++++++++++++++ pkg/router/metrics/metrics.go | 6 ++++-- 4 files changed, 48 insertions(+), 15 deletions(-) diff --git a/pkg/cmd/infra/router/template.go b/pkg/cmd/infra/router/template.go index 1e81c0a4c2d7..10956fd2fce5 100644 --- a/pkg/cmd/infra/router/template.go +++ b/pkg/cmd/infra/router/template.go @@ -243,6 +243,7 @@ func (o *TemplateRouterOptions) Validate() error { // Run launches a template router using the provided options. It never exits. func (o *TemplateRouterOptions) Run() error { glog.Infof("Starting template router (%s)", version.Get()) + var ptrTemplatePlugin *templateplugin.TemplatePlugin var reloadCallbacks []func() @@ -310,9 +311,15 @@ func (o *TemplateRouterOptions) Run() error { if err != nil { return fmt.Errorf("ROUTER_METRICS_READY_HTTP_URL must be a valid URL or empty: %v", err) } - check := metrics.HTTPBackendAvailable(u) + checkBackend := metrics.HTTPBackendAvailable(u) if isTrue(util.Env("ROUTER_USE_PROXY_PROTOCOL", "")) { - check = metrics.ProxyProtocolHTTPBackendAvailable(u) + checkBackend = metrics.ProxyProtocolHTTPBackendAvailable(u) + } + checkSync := metrics.HasSynced(&ptrTemplatePlugin) + checkController := metrics.ControllerLive() + liveChecks := []healthz.HealthzChecker{checkController} + if !(isTrue(util.Env("ROUTER_BIND_PORTS_BEFORE_SYNC", ""))) { + liveChecks = append(liveChecks, checkBackend) } kubeconfig := o.Config.KubeConfig() @@ -353,7 +360,8 @@ func (o *TemplateRouterOptions) Run() error { Resource: "routers", Name: o.RouterName, }, - Checks: []healthz.HealthzChecker{check}, + LiveChecks: liveChecks, + ReadyChecks: []healthz.HealthzChecker{checkBackend, checkSync}, } if certFile := util.Env("ROUTER_METRICS_TLS_CERT_FILE", ""); len(certFile) > 0 { certificate, err := tls.LoadX509KeyPair(certFile, util.Env("ROUTER_METRICS_TLS_KEY_FILE", "")) @@ -411,6 +419,7 @@ func (o *TemplateRouterOptions) Run() error { if err != nil { return err } + ptrTemplatePlugin = templatePlugin factory := o.RouterSelection.NewFactory(routeclient, projectclient.Project().Projects(), kc) factory.RouteModifierFn = o.RouteUpdate diff --git a/pkg/oc/admin/router/router.go b/pkg/oc/admin/router/router.go index 47241e6e3fe2..704e140f966f 100644 --- a/pkg/oc/admin/router/router.go +++ b/pkg/oc/admin/router/router.go @@ -243,9 +243,8 @@ const ( // Default port numbers to expose and bind/listen on. defaultPorts = "80:80,443:443" - // Default stats and healthz port. - defaultStatsPort = 1936 - defaultHealthzPort = defaultStatsPort + // Default stats port. + defaultStatsPort = 1936 ) // NewCmdRouter implements the OpenShift CLI router command. @@ -436,21 +435,21 @@ func generateSecretsConfig(cfg *RouterConfig, namespace string, defaultCert []by return secrets, volumes, mounts, nil } -func generateProbeConfigForRouter(cfg *RouterConfig, ports []kapi.ContainerPort) *kapi.Probe { +func generateProbeConfigForRouter(path string, cfg *RouterConfig, ports []kapi.ContainerPort) *kapi.Probe { var probe *kapi.Probe if cfg.Type == "haproxy-router" { probe = &kapi.Probe{} - healthzPort := defaultHealthzPort + probePort := defaultStatsPort if cfg.StatsPort > 0 { - healthzPort = cfg.StatsPort + probePort = cfg.StatsPort } probe.Handler.HTTPGet = &kapi.HTTPGetAction{ - Path: "/healthz", + Path: path, Port: intstr.IntOrString{ Type: intstr.Int, - IntVal: int32(healthzPort), + IntVal: int32(probePort), }, } @@ -466,7 +465,7 @@ func generateProbeConfigForRouter(cfg *RouterConfig, ports []kapi.ContainerPort) } func generateLivenessProbeConfig(cfg *RouterConfig, ports []kapi.ContainerPort) *kapi.Probe { - probe := generateProbeConfigForRouter(cfg, ports) + probe := generateProbeConfigForRouter("/healthz", cfg, ports) if probe != nil { probe.InitialDelaySeconds = 10 } @@ -474,7 +473,7 @@ func generateLivenessProbeConfig(cfg *RouterConfig, ports []kapi.ContainerPort) } func generateReadinessProbeConfig(cfg *RouterConfig, ports []kapi.ContainerPort) *kapi.Probe { - probe := generateProbeConfigForRouter(cfg, ports) + probe := generateProbeConfigForRouter("healthz/ready", cfg, ports) if probe != nil { probe.InitialDelaySeconds = 10 } diff --git a/pkg/router/metrics/health.go b/pkg/router/metrics/health.go index dc99dbada8cd..3faebf1e0561 100644 --- a/pkg/router/metrics/health.go +++ b/pkg/router/metrics/health.go @@ -12,6 +12,7 @@ import ( "github.com/golang/glog" + templateplugin "github.com/openshift/origin/pkg/router/template" "k8s.io/apiserver/pkg/server/healthz" "k8s.io/kubernetes/pkg/probe" probehttp "k8s.io/kubernetes/pkg/probe/http" @@ -35,6 +36,28 @@ func HTTPBackendAvailable(u *url.URL) healthz.HealthzChecker { }) } +// HasSynced returns a healthz check that verifies the router has been synced at least +// once. +func HasSynced(router **templateplugin.TemplatePlugin) healthz.HealthzChecker { + return healthz.NamedCheck("has-synced", func(r *http.Request) error { + if router != nil { + if (*router).Router.SyncedAtLeastOnce() == true { + return nil + } else { + return fmt.Errorf("Router not synced") + } + } + return nil + }) +} + +func ControllerLive() healthz.HealthzChecker { + return healthz.NamedCheck("controller", func(r *http.Request) error { + return nil + }) + +} + // ProxyProtocolHTTPBackendAvailable returns a healthz check that verifies a backend supporting // the HAProxy PROXY protocol responds to a GET to the provided URL with 2xx or 3xx response. func ProxyProtocolHTTPBackendAvailable(u *url.URL) healthz.HealthzChecker { diff --git a/pkg/router/metrics/metrics.go b/pkg/router/metrics/metrics.go index 840c72bd2fe2..1f27afb30260 100644 --- a/pkg/router/metrics/metrics.go +++ b/pkg/router/metrics/metrics.go @@ -31,12 +31,14 @@ type Listener struct { Authorizer authorizer.Authorizer Record authorizer.AttributesRecord - Checks []healthz.HealthzChecker + LiveChecks []healthz.HealthzChecker + ReadyChecks []healthz.HealthzChecker } func (l Listener) handler() http.Handler { mux := http.NewServeMux() - healthz.InstallHandler(mux, l.Checks...) + healthz.InstallHandler(mux, l.LiveChecks...) + healthz.InstallPathHandler(mux, "/healthz/ready", l.ReadyChecks...) if l.Authenticator != nil { protected := http.NewServeMux()