Merge pull request #17861 from deads2k/controller-16-wait-3.9

Automatic merge from submit-queue.

 run openshift and kube controllers on different leases

In 3.7, we started using secondary leases for the scheduler and the kube controllers.  In the 3.9 rebase, we separate informers.  This pull completes the leasing and running separation.

@smarterclayton @soltysh 
/assign @soltysh

pkg/cmd/server/start/start_kube_controller_manager.go

@@ -34,7 +34,8 @@ func newKubeControllerContext(informers *informers) func(s *controlleroptions.CM
 			return controllerapp.ControllerContext{}, err
 		}
 
-		// Overwrite the informers, because we have our custom generic informers.
+		// Overwrite the informers, because we have our custom generic informers for quota.
+		// TODO update quota to create its own informer like garbage collection or if we split this out, actually add our external types to the kube generic informer
 		ret.InformerFactory = externalKubeInformersWithExtraGenerics{
 			SharedInformerFactory:   informers.GetExternalKubeInformers(),
 			genericResourceInformer: informers.ToGenericInformer(),
@@ -225,9 +226,6 @@ func createRecylerTemplate(recyclerImage string) (string, error) {
 func runEmbeddedKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout string, dynamicProvisioningEnabled bool, cmdLineArgs map[string][]string,
 	recyclerImage string, informers *informers) {
 	controllerapp.CreateControllerContext = newKubeControllerContext(informers)
-	controllerapp.StartInformers = func(stop <-chan struct{}) {
-		informers.Start(stop)
-	}
 
 	// TODO we need a real identity for this.  Right now it's just using the loopback connection like it used to.
 	controllerManager, cleanupFunctions, err := newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, recyclerImage, dynamicProvisioningEnabled, cmdLineArgs)
@@ -258,3 +256,8 @@ type externalKubeInformersWithExtraGenerics struct {
 func (i externalKubeInformersWithExtraGenerics) ForResource(resource schema.GroupVersionResource) (kinformers.GenericInformer, error) {
 	return i.genericResourceInformer.ForResource(resource)
 }
+
+func (i externalKubeInformersWithExtraGenerics) Start(stopCh <-chan struct{}) {
+	i.SharedInformerFactory.Start(stopCh)
+	i.genericResourceInformer.Start(stopCh)
+}

pkg/cmd/server/start/start_master.go

@@ -391,10 +391,6 @@ func (m *Master) Start() error {
 		if err != nil {
 			return err
 		}
-		kubeControllerInformers, err := NewInformers(*m.config)
-		if err != nil {
-			return err
-		}
 
 		_, config, err := configapi.GetExternalKubeClient(m.config.MasterClients.OpenShiftLoopbackKubeConfig, m.config.MasterClients.OpenShiftLoopbackClientConnectionOverrides)
 		if err != nil {
@@ -470,38 +466,42 @@ func (m *Master) Start() error {
 			glog.Fatalf("Controller graceful shutdown requested")
 		}()
 
+		go runEmbeddedScheduler(m.config.MasterClients.OpenShiftLoopbackKubeConfig, m.config.KubernetesMasterConfig.SchedulerConfigFile, m.config.KubernetesMasterConfig.SchedulerArguments)
+
+		kubeControllerInformers, err := NewInformers(*m.config)
+		if err != nil {
+			return err
+		}
+		go runEmbeddedKubeControllerManager(
+			m.config.MasterClients.OpenShiftLoopbackKubeConfig,
+			m.config.ServiceAccountConfig.PrivateKeyFile,
+			m.config.ServiceAccountConfig.MasterCA,
+			m.config.KubernetesMasterConfig.PodEvictionTimeout,
+			m.config.VolumeConfig.DynamicProvisioningEnabled,
+			m.config.KubernetesMasterConfig.ControllerArguments,
+			recyclerImage,
+			kubeControllerInformers)
+
 		go func() {
 			controllerPlug.WaitForStart()
 			if err := waitForHealthyAPIServer(kubeExternal.Discovery().RESTClient()); err != nil {
 				glog.Fatal(err)
 			}
 
-			// continuously run the scheduler while we have the primary lease
-			go runEmbeddedScheduler(m.config.MasterClients.OpenShiftLoopbackKubeConfig, m.config.KubernetesMasterConfig.SchedulerConfigFile, m.config.KubernetesMasterConfig.SchedulerArguments)
-
-			go runEmbeddedKubeControllerManager(
-				m.config.MasterClients.OpenShiftLoopbackKubeConfig,
-				m.config.ServiceAccountConfig.PrivateKeyFile,
-				m.config.ServiceAccountConfig.MasterCA,
-				m.config.KubernetesMasterConfig.PodEvictionTimeout,
-				m.config.VolumeConfig.DynamicProvisioningEnabled,
-				m.config.KubernetesMasterConfig.ControllerArguments,
-				recyclerImage,
-				kubeControllerInformers)
-
 			openshiftControllerOptions, err := getOpenshiftControllerOptions(m.config.KubernetesMasterConfig.ControllerArguments)
 			if err != nil {
 				glog.Fatal(err)
 			}
 
-			controllerContext := newControllerContext(openshiftControllerOptions, m.config.ControllerConfig.Controllers, privilegedLoopbackConfig, kubeExternal, openshiftControllerInformers, utilwait.NeverStop, make(chan struct{}))
+			stopCh := utilwait.NeverStop
+			informersStarted := make(chan struct{})
+			controllerContext := newControllerContext(openshiftControllerOptions, m.config.ControllerConfig.Controllers, privilegedLoopbackConfig, kubeExternal, openshiftControllerInformers, stopCh, informersStarted)
 			if err := startControllers(*m.config, allocationController, controllerContext); err != nil {
 				glog.Fatal(err)
 			}
 
-			openshiftControllerInformers.Start(utilwait.NeverStop)
-			close(controllerContext.InformersStarted)
-
+			openshiftControllerInformers.Start(stopCh)
+			close(informersStarted)
 		}()
 	}