1.12.2 rebase

.gitignore

@@ -110,14 +110,11 @@ kubernetes.tar.gz
 # generated files in any directory
 # TODO(thockin): uncomment this when we stop committing the generated files.
 #zz_generated.*
-zz_generated.openapi.go
-zz_generated_*_test.go
 
 # make-related metadata
 /.make/
 
 # Just in time generated data in the source, should never be committed
-/test/e2e/generated/bindata.go
 
 # This file used by some vendor repos (e.g. github.com/go-openapi/...) to store secret variables and should not be ignored
 !\.drone\.sec

build/root/Makefile

@@ -603,3 +603,11 @@ else
 bazel-release:
 	bazel build //build/release-tars
 endif
+
+.PHONY: fork-build
+fork-build: generated_files
+	GOFLAGS="-tags kubernetes" hack/make-rules/build.sh $(WHAT)
+
+.PHONY: fork-test
+fork-test: generated_files
+	GOFLAGS="-tags kubernetes" hack/make-rules/test.sh $(WHAT) $(TESTS)

cmd/kube-apiserver/app/patch_openshift.go

@@ -0,0 +1,35 @@
+package app
+
+import (
+	"k8s.io/apiserver/pkg/admission"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	clientgoinformers "k8s.io/client-go/informers"
+	informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
+	"k8s.io/kubernetes/pkg/master"
+)
+
+type KubeAPIServerConfigFunc func(config *genericapiserver.Config, sharedInformers informers.SharedInformerFactory, versionedInformers clientgoinformers.SharedInformerFactory, pluginInitializers *[]admission.PluginInitializer) (genericapiserver.DelegationTarget, error)
+
+var OpenShiftKubeAPIServerConfigPatch KubeAPIServerConfigFunc = nil
+
+type KubeAPIServerServerFunc func(server *master.Master) error
+
+func PatchKubeAPIServerConfig(config *genericapiserver.Config, sharedInformers informers.SharedInformerFactory, versionedInformers clientgoinformers.SharedInformerFactory, pluginInitializers *[]admission.PluginInitializer) (genericapiserver.DelegationTarget, error) {
+	if OpenShiftKubeAPIServerConfigPatch == nil {
+		return genericapiserver.NewEmptyDelegate(), nil
+	}
+
+	return OpenShiftKubeAPIServerConfigPatch(config, sharedInformers, versionedInformers, pluginInitializers)
+}
+
+var OpenShiftKubeAPIServerServerPatch KubeAPIServerServerFunc = nil
+
+func PatchKubeAPIServerServer(server *master.Master) error {
+	if OpenShiftKubeAPIServerServerPatch == nil {
+		return nil
+	}
+
+	return OpenShiftKubeAPIServerServerPatch(server)
+}
+
+var StartingDelegate genericapiserver.DelegationTarget = genericapiserver.NewEmptyDelegate()

cmd/kube-apiserver/app/server.go

@@ -176,7 +176,7 @@ func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan
 	if err != nil {
 		return nil, err
 	}
-	apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, genericapiserver.NewEmptyDelegate())
+	apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, StartingDelegate)
 	if err != nil {
 		return nil, err
 	}
@@ -186,6 +186,10 @@ func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan
 		return nil, err
 	}
 
+	if err := PatchKubeAPIServerServer(kubeAPIServer); err != nil {
+		return nil, err
+	}
+
 	// otherwise go down the normal path of standing the aggregator up in front of the API server
 	// this wires up openapi
 	kubeAPIServer.GenericAPIServer.PrepareRun()
@@ -421,6 +425,7 @@ func CreateKubeAPIServerConfig(
 func buildGenericConfig(
 	s *options.ServerRunOptions,
 	proxyTransport *http.Transport,
+
 ) (
 	genericConfig *genericapiserver.Config,
 	sharedInformers informers.SharedInformerFactory,
@@ -574,6 +579,12 @@ func buildGenericConfig(
 		return
 	}
 
+	StartingDelegate, err = PatchKubeAPIServerConfig(genericConfig, sharedInformers, versionedInformers, &pluginInitializers)
+	if err != nil {
+		lastErr = fmt.Errorf("failed to patch: %v", err)
+		return
+	}
+
 	err = s.Admission.ApplyTo(
 		genericConfig,
 		versionedInformers,

cmd/kube-controller-manager/app/apps.go

@@ -37,7 +37,10 @@ func startDaemonSetController(ctx ControllerContext) (http.Handler, bool, error)
 	if !ctx.AvailableResources[schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "daemonsets"}] {
 		return nil, false, nil
 	}
-	dsc, err := daemon.NewDaemonSetsController(
+	dsc, err := daemon.NewNodeSelectorAwareDaemonSetsController(
+		ctx.OpenShiftContext.OpenShiftDefaultProjectNodeSelector,
+		ctx.OpenShiftContext.KubeDefaultProjectNodeSelector,
+		ctx.InformerFactory.Core().V1().Namespaces(),
 		ctx.InformerFactory.Apps().V1().DaemonSets(),
 		ctx.InformerFactory.Apps().V1().ControllerRevisions(),
 		ctx.InformerFactory.Core().V1().Pods(),

cmd/kube-controller-manager/app/config/config.go

@@ -26,6 +26,8 @@ import (
 
 // Config is the main context object for the controller manager.
 type Config struct {
+	OpenShiftContext OpenShiftContext
+
 	ComponentConfig kubectrlmgrconfig.KubeControllerManagerConfiguration
 
 	SecureServing *apiserver.SecureServingInfo

cmd/kube-controller-manager/app/config/patch.go

@@ -0,0 +1,9 @@
+package config
+
+// OpenShiftContext is additional context that we need to launch the kube-controller-manager for openshift.
+// Basically, this holds our additional config information.
+type OpenShiftContext struct {
+	OpenShiftConfig                     string
+	OpenShiftDefaultProjectNodeSelector string
+	KubeDefaultProjectNodeSelector      string
+}

cmd/kube-controller-manager/app/controllermanager.go

@@ -100,6 +100,13 @@ controller, and serviceaccounts controller.`,
 				os.Exit(1)
 			}
 
+			cleanupFn, err := ShimForOpenShift(s, c)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+				os.Exit(1)
+			}
+			defer cleanupFn()
+
 			if err := Run(c.Complete(), wait.NeverStop); err != nil {
 				fmt.Fprintf(os.Stderr, "%v\n", err)
 				os.Exit(1)
@@ -193,6 +200,10 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
 		}
 		saTokenControllerInitFunc := serviceAccountTokenControllerStarter{rootClientBuilder: rootClientBuilder}.startServiceAccountTokenController
 
+		if err := createPVRecyclerSA(c.OpenShiftContext.OpenShiftConfig, rootClientBuilder); err != nil {
+			glog.Fatalf("error creating recycler serviceaccount: %v", err)
+		}
+
 		if err := StartControllers(controllerContext, saTokenControllerInitFunc, NewControllerInitializers(controllerContext.LoopMode), unsecuredMux); err != nil {
 			glog.Fatalf("error starting controllers: %v", err)
 		}
@@ -243,6 +254,8 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
 }
 
 type ControllerContext struct {
+	OpenShiftContext config.OpenShiftContext
+
 	// ClientBuilder will provide a client for this controller to use
 	ClientBuilder controller.ControllerClientBuilder
 
@@ -416,7 +429,12 @@ func GetAvailableResources(clientBuilder controller.ControllerClientBuilder) (ma
 // the shared-informers client and token controller.
 func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}) (ControllerContext, error) {
 	versionedClient := rootClientBuilder.ClientOrDie("shared-informers")
-	sharedInformers := informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)())
+	var sharedInformers informers.SharedInformerFactory
+	if InformerFactoryOverride == nil {
+		sharedInformers = informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)())
+	} else {
+		sharedInformers = InformerFactoryOverride
+	}
 
 	// If apiserver is not running we should wait for some time and fail only then. This is particularly
 	// important when we start apiserver and controller manager at the same time.
@@ -444,6 +462,7 @@ func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clien
 	}
 
 	ctx := ControllerContext{
+		OpenShiftContext:   s.OpenShiftContext,
 		ClientBuilder:      clientBuilder,
 		InformerFactory:    sharedInformers,
 		ComponentConfig:    s.ComponentConfig,
@@ -543,10 +562,10 @@ func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController
 		ctx.InformerFactory.Core().V1().ServiceAccounts(),
 		ctx.InformerFactory.Core().V1().Secrets(),
 		c.rootClientBuilder.ClientOrDie("tokens-controller"),
-		serviceaccountcontroller.TokensControllerOptions{
+		applyOpenShiftServiceServingCertCA(serviceaccountcontroller.TokensControllerOptions{
 			TokenGenerator: tokenGenerator,
 			RootCA:         rootCA,
-		},
+		}),
 	)
 	if err != nil {
 		return nil, true, fmt.Errorf("error creating Tokens controller: %v", err)

cmd/kube-controller-manager/app/options/options.go

@@ -85,8 +85,9 @@ type KubeControllerManagerOptions struct {
 	Authentication  *apiserveroptions.DelegatingAuthenticationOptions
 	Authorization   *apiserveroptions.DelegatingAuthorizationOptions
 
-	Master     string
-	Kubeconfig string
+	Master           string
+	Kubeconfig       string
+	OpenShiftContext kubecontrollerconfig.OpenShiftContext
 }
 
 // NewKubeControllerManagerOptions creates a new KubeControllerManagerOptions with a default config.
@@ -263,6 +264,8 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy
 	var dummy string
 	fs.MarkDeprecated("insecure-experimental-approve-all-kubelet-csrs-for-group", "This flag does nothing.")
 	fs.StringVar(&dummy, "insecure-experimental-approve-all-kubelet-csrs-for-group", "", "This flag does nothing.")
+	fs.StringVar(&s.OpenShiftContext.OpenShiftConfig, "openshift-config", s.OpenShiftContext.OpenShiftConfig, "indicates that this process should be compatible with openshift start master")
+	fs.MarkHidden("openshift-config")
 	utilfeature.DefaultFeatureGate.AddFlag(fss.FlagSet("generic"))
 
 	return fss
@@ -356,6 +359,8 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) e
 	c.ComponentConfig.Generic.Port = int32(s.InsecureServing.BindPort)
 	c.ComponentConfig.Generic.Address = s.InsecureServing.BindAddress.String()
 
+	c.OpenShiftContext = s.OpenShiftContext
+
 	return nil
 }
 

cmd/kube-controller-manager/app/patch.go

@@ -0,0 +1,49 @@
+package app
+
+import (
+	"path"
+
+	"k8s.io/client-go/informers"
+	"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
+	"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
+)
+
+var InformerFactoryOverride informers.SharedInformerFactory
+
+func ShimForOpenShift(controllerManagerOptions *options.KubeControllerManagerOptions, controllerManager *config.Config) (func(), error) {
+	if len(controllerManager.OpenShiftContext.OpenShiftConfig) == 0 {
+		return func() {}, nil
+	}
+
+	// TODO this gets removed when no longer take flags and no longer build a recycler template
+	openshiftConfig, err := getOpenShiftConfig(controllerManager.OpenShiftContext.OpenShiftConfig)
+	if err != nil {
+		return func() {}, err
+	}
+	// apply the config based controller manager flags.  They will override.
+	// TODO this should be replaced by the installer setting up the flags for us
+	if err := applyOpenShiftConfigFlags(controllerManagerOptions, controllerManager, openshiftConfig); err != nil {
+		return func() {}, err
+	}
+
+	// TODO this should be replaced by using a flex volume to inject service serving cert CAs into pods instead of adding it to the sa token
+	if err := applyOpenShiftServiceServingCertCAFunc(path.Dir(controllerManager.OpenShiftContext.OpenShiftConfig), openshiftConfig); err != nil {
+		return func() {}, err
+	}
+
+	// skip GC on some openshift resources
+	// TODO this should be replaced by discovery information in some way
+	if err := applyOpenShiftGCConfig(controllerManager); err != nil {
+		return func() {}, err
+	}
+
+	// Overwrite the informers, because we have our custom generic informers for quota.
+	// TODO update quota to create its own informer like garbage collection
+	if informers, err := newInformerFactory(controllerManager.Kubeconfig); err != nil {
+		return func() {}, err
+	} else {
+		InformerFactoryOverride = informers
+	}
+
+	return func() {}, nil
+}