Perform real backoff when contending for writes from the router

The current route backoff mechanism works by tracking the last touch
time from other routes, but this is prone to failure and will not scale
to very large sets of routers competing for updating status.

Instead, treat the ability to write status as a lease renewal, and have
failure to write status as a cue to backoff. Each new write further
increases the lease confidence up to an interval. Treat observed writes
from other processes as a signal that the lease holder is maintaining
their lease.

pkg/cmd/infra/router/f5.go

@@ -4,11 +4,13 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
 	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
 
@@ -21,6 +23,7 @@ import (
 	"github.com/openshift/origin/pkg/router"
 	"github.com/openshift/origin/pkg/router/controller"
 	f5plugin "github.com/openshift/origin/pkg/router/f5"
+	"github.com/openshift/origin/pkg/util/writerlease"
 	"github.com/openshift/origin/pkg/version"
 )
 
@@ -228,10 +231,14 @@ func (o *F5RouterOptions) Run() error {
 		return err
 	}
 
-	var recorder controller.RejectionRecorder = controller.LogRejections
 	var plugin router.Plugin = f5Plugin
+	var recorder controller.RejectionRecorder = controller.LogRejections
 	if o.UpdateStatus {
-		status := controller.NewStatusAdmitter(plugin, routeclient.Route(), o.RouterName, o.RouterCanonicalHostname)
+		lease := writerlease.New(time.Minute, time.Second)
+		tracker := controller.NewSimpleContentionTracker(o.ResyncInterval / 10)
+		go lease.Run(wait.NeverStop)
+		go tracker.Run(wait.NeverStop)
+		status := controller.NewStatusAdmitter(plugin, routeclient.Route(), o.RouterName, o.RouterCanonicalHostname, lease, tracker)
 		recorder = status
 		plugin = status
 	}

pkg/cmd/infra/router/router.go

@@ -138,7 +138,7 @@ func (o *RouterSelection) AdmissionCheck(route *routeapi.Route) error {
 		return fmt.Errorf("host not in the allowed list of domains")
 	}
 
-	glog.V(4).Infof("host %s admitted", route.Spec.Host)
+	glog.V(4).Infof("host %s passed admission", route.Spec.Host)
 	return nil
 }
 

pkg/cmd/infra/router/template.go

@@ -17,6 +17,7 @@ import (
 
 	ktypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
@@ -38,6 +39,7 @@ import (
 	"github.com/openshift/origin/pkg/router/metrics/haproxy"
 	templateplugin "github.com/openshift/origin/pkg/router/template"
 	"github.com/openshift/origin/pkg/util/proc"
+	"github.com/openshift/origin/pkg/util/writerlease"
 	"github.com/openshift/origin/pkg/version"
 )
 
@@ -404,16 +406,20 @@ func (o *TemplateRouterOptions) Run() error {
 		return err
 	}
 
-	svcFetcher := templateplugin.NewListWatchServiceLookup(kc.Core(), 10*time.Minute)
+	svcFetcher := templateplugin.NewListWatchServiceLookup(kc.Core(), o.ResyncInterval)
 	templatePlugin, err := templateplugin.NewTemplatePlugin(pluginCfg, svcFetcher)
 	if err != nil {
 		return err
 	}
 
-	var recorder controller.RejectionRecorder = controller.LogRejections
 	var plugin router.Plugin = templatePlugin
+	var recorder controller.RejectionRecorder = controller.LogRejections
 	if o.UpdateStatus {
-		status := controller.NewStatusAdmitter(plugin, routeclient.Route(), o.RouterName, o.RouterCanonicalHostname)
+		lease := writerlease.New(time.Minute, time.Second)
+		tracker := controller.NewSimpleContentionTracker(o.ResyncInterval / 10)
+		go lease.Run(wait.NeverStop)
+		go tracker.Run(wait.NeverStop)
+		status := controller.NewStatusAdmitter(plugin, routeclient.Route(), o.RouterName, o.RouterCanonicalHostname, lease, tracker)
 		recorder = status
 		plugin = status
 	}