diff --git a/pkg/cmd/infra/router/template.go b/pkg/cmd/infra/router/template.go index 1e81c0a4c2d7..10956fd2fce5 100644 --- a/pkg/cmd/infra/router/template.go +++ b/pkg/cmd/infra/router/template.go @@ -243,6 +243,7 @@ func (o *TemplateRouterOptions) Validate() error { // Run launches a template router using the provided options. It never exits. func (o *TemplateRouterOptions) Run() error { glog.Infof("Starting template router (%s)", version.Get()) + var ptrTemplatePlugin *templateplugin.TemplatePlugin var reloadCallbacks []func() @@ -310,9 +311,15 @@ func (o *TemplateRouterOptions) Run() error { if err != nil { return fmt.Errorf("ROUTER_METRICS_READY_HTTP_URL must be a valid URL or empty: %v", err) } - check := metrics.HTTPBackendAvailable(u) + checkBackend := metrics.HTTPBackendAvailable(u) if isTrue(util.Env("ROUTER_USE_PROXY_PROTOCOL", "")) { - check = metrics.ProxyProtocolHTTPBackendAvailable(u) + checkBackend = metrics.ProxyProtocolHTTPBackendAvailable(u) + } + checkSync := metrics.HasSynced(&ptrTemplatePlugin) + checkController := metrics.ControllerLive() + liveChecks := []healthz.HealthzChecker{checkController} + if !(isTrue(util.Env("ROUTER_BIND_PORTS_BEFORE_SYNC", ""))) { + liveChecks = append(liveChecks, checkBackend) } kubeconfig := o.Config.KubeConfig() @@ -353,7 +360,8 @@ func (o *TemplateRouterOptions) Run() error { Resource: "routers", Name: o.RouterName, }, - Checks: []healthz.HealthzChecker{check}, + LiveChecks: liveChecks, + ReadyChecks: []healthz.HealthzChecker{checkBackend, checkSync}, } if certFile := util.Env("ROUTER_METRICS_TLS_CERT_FILE", ""); len(certFile) > 0 { certificate, err := tls.LoadX509KeyPair(certFile, util.Env("ROUTER_METRICS_TLS_KEY_FILE", "")) @@ -411,6 +419,7 @@ func (o *TemplateRouterOptions) Run() error { if err != nil { return err } + ptrTemplatePlugin = templatePlugin factory := o.RouterSelection.NewFactory(routeclient, projectclient.Project().Projects(), kc) factory.RouteModifierFn = o.RouteUpdate diff --git a/pkg/oc/admin/router/router.go b/pkg/oc/admin/router/router.go index 47241e6e3fe2..704e140f966f 100644 --- a/pkg/oc/admin/router/router.go +++ b/pkg/oc/admin/router/router.go @@ -243,9 +243,8 @@ const ( // Default port numbers to expose and bind/listen on. defaultPorts = "80:80,443:443" - // Default stats and healthz port. - defaultStatsPort = 1936 - defaultHealthzPort = defaultStatsPort + // Default stats port. + defaultStatsPort = 1936 ) // NewCmdRouter implements the OpenShift CLI router command. @@ -436,21 +435,21 @@ func generateSecretsConfig(cfg *RouterConfig, namespace string, defaultCert []by return secrets, volumes, mounts, nil } -func generateProbeConfigForRouter(cfg *RouterConfig, ports []kapi.ContainerPort) *kapi.Probe { +func generateProbeConfigForRouter(path string, cfg *RouterConfig, ports []kapi.ContainerPort) *kapi.Probe { var probe *kapi.Probe if cfg.Type == "haproxy-router" { probe = &kapi.Probe{} - healthzPort := defaultHealthzPort + probePort := defaultStatsPort if cfg.StatsPort > 0 { - healthzPort = cfg.StatsPort + probePort = cfg.StatsPort } probe.Handler.HTTPGet = &kapi.HTTPGetAction{ - Path: "/healthz", + Path: path, Port: intstr.IntOrString{ Type: intstr.Int, - IntVal: int32(healthzPort), + IntVal: int32(probePort), }, } @@ -466,7 +465,7 @@ func generateProbeConfigForRouter(cfg *RouterConfig, ports []kapi.ContainerPort) } func generateLivenessProbeConfig(cfg *RouterConfig, ports []kapi.ContainerPort) *kapi.Probe { - probe := generateProbeConfigForRouter(cfg, ports) + probe := generateProbeConfigForRouter("/healthz", cfg, ports) if probe != nil { probe.InitialDelaySeconds = 10 } @@ -474,7 +473,7 @@ func generateLivenessProbeConfig(cfg *RouterConfig, ports []kapi.ContainerPort) } func generateReadinessProbeConfig(cfg *RouterConfig, ports []kapi.ContainerPort) *kapi.Probe { - probe := generateProbeConfigForRouter(cfg, ports) + probe := generateProbeConfigForRouter("healthz/ready", cfg, ports) if probe != nil { probe.InitialDelaySeconds = 10 } diff --git a/pkg/router/metrics/health.go b/pkg/router/metrics/health.go index dc99dbada8cd..3faebf1e0561 100644 --- a/pkg/router/metrics/health.go +++ b/pkg/router/metrics/health.go @@ -12,6 +12,7 @@ import ( "github.com/golang/glog" + templateplugin "github.com/openshift/origin/pkg/router/template" "k8s.io/apiserver/pkg/server/healthz" "k8s.io/kubernetes/pkg/probe" probehttp "k8s.io/kubernetes/pkg/probe/http" @@ -35,6 +36,28 @@ func HTTPBackendAvailable(u *url.URL) healthz.HealthzChecker { }) } +// HasSynced returns a healthz check that verifies the router has been synced at least +// once. +func HasSynced(router **templateplugin.TemplatePlugin) healthz.HealthzChecker { + return healthz.NamedCheck("has-synced", func(r *http.Request) error { + if router != nil { + if (*router).Router.SyncedAtLeastOnce() == true { + return nil + } else { + return fmt.Errorf("Router not synced") + } + } + return nil + }) +} + +func ControllerLive() healthz.HealthzChecker { + return healthz.NamedCheck("controller", func(r *http.Request) error { + return nil + }) + +} + // ProxyProtocolHTTPBackendAvailable returns a healthz check that verifies a backend supporting // the HAProxy PROXY protocol responds to a GET to the provided URL with 2xx or 3xx response. func ProxyProtocolHTTPBackendAvailable(u *url.URL) healthz.HealthzChecker { diff --git a/pkg/router/metrics/metrics.go b/pkg/router/metrics/metrics.go index 840c72bd2fe2..1f27afb30260 100644 --- a/pkg/router/metrics/metrics.go +++ b/pkg/router/metrics/metrics.go @@ -31,12 +31,14 @@ type Listener struct { Authorizer authorizer.Authorizer Record authorizer.AttributesRecord - Checks []healthz.HealthzChecker + LiveChecks []healthz.HealthzChecker + ReadyChecks []healthz.HealthzChecker } func (l Listener) handler() http.Handler { mux := http.NewServeMux() - healthz.InstallHandler(mux, l.Checks...) + healthz.InstallHandler(mux, l.LiveChecks...) + healthz.InstallPathHandler(mux, "/healthz/ready", l.ReadyChecks...) if l.Authenticator != nil { protected := http.NewServeMux() diff --git a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go index bd8bfe7b53d9..224f1eda2c44 100644 --- a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go +++ b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go @@ -66,6 +66,15 @@ func NamedCheck(name string, check func(r *http.Request) error) HealthzChecker { // exactly one call to InstallHandler. Calling InstallHandler more // than once for the same mux will result in a panic. func InstallHandler(mux mux, checks ...HealthzChecker) { + InstallPathHandler(mux, "/healthz", checks...) +} + +// InstallPathHandler registers handlers for health checking on +// a specific path to mux. *All handlers* for the path must be +// specified in exactly one call to InstallPathHandler. Calling +// InstallPathHandler more than once for the same path and mux will +// result in a panic. +func InstallPathHandler(mux mux, path string, checks ...HealthzChecker) { if len(checks) == 0 { glog.V(5).Info("No default health checks specified. Installing the ping handler.") checks = []HealthzChecker{PingHealthz} @@ -73,9 +82,9 @@ func InstallHandler(mux mux, checks ...HealthzChecker) { glog.V(5).Info("Installing healthz checkers:", strings.Join(checkerNames(checks...), ", ")) - mux.Handle("/healthz", handleRootHealthz(checks...)) + mux.Handle(path, handleRootHealthz(checks...)) for _, check := range checks { - mux.Handle(fmt.Sprintf("/healthz/%v", check.Name()), adaptCheckToHandler(check.Check)) + mux.Handle(fmt.Sprintf("%s/%v", path, check.Name()), adaptCheckToHandler(check.Check)) } } diff --git a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go index 99aae5a9eabd..a1f761b786b0 100644 --- a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go +++ b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go @@ -42,6 +42,38 @@ func TestInstallHandler(t *testing.T) { } } +func TestInstallPathHandler(t *testing.T) { + mux := http.NewServeMux() + InstallPathHandler(mux, "/healthz/test") + InstallPathHandler(mux, "/healthz/ready") + req, err := http.NewRequest("GET", "http://example.com/healthz/test", nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("expected %v, got %v", http.StatusOK, w.Code) + } + if w.Body.String() != "ok" { + t.Errorf("expected %v, got %v", "ok", w.Body.String()) + } + + req, err = http.NewRequest("GET", "http://example.com/healthz/ready", nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + w = httptest.NewRecorder() + mux.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("expected %v, got %v", http.StatusOK, w.Code) + } + if w.Body.String() != "ok" { + t.Errorf("expected %v, got %v", "ok", w.Body.String()) + } + +} + func TestMulitipleChecks(t *testing.T) { tests := []struct { path string