From bb1d044b92dbf6abc05c68c0020b5e850541ae9a Mon Sep 17 00:00:00 2001 From: David Eads Date: Wed, 7 Feb 2018 11:56:01 -0500 Subject: [PATCH] simplify openshift controller manager startup --- pkg/cmd/server/origin/controller/config.go | 62 ----------------- .../origin/controller/serviceaccount.go | 33 --------- .../start/start_kube_controller_manager.go | 2 - pkg/cmd/server/start/start_master.go | 68 +++++++------------ 4 files changed, 23 insertions(+), 142 deletions(-) diff --git a/pkg/cmd/server/origin/controller/config.go b/pkg/cmd/server/origin/controller/config.go index 5d2ee4bbd543..1895768b0f77 100644 --- a/pkg/cmd/server/origin/controller/config.go +++ b/pkg/cmd/server/origin/controller/config.go @@ -1,20 +1,15 @@ package controller import ( - "fmt" - "io/ioutil" "path" "time" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/util/cert" "k8s.io/kubernetes/pkg/api/legacyscheme" kapi "k8s.io/kubernetes/pkg/apis/core" - kcontroller "k8s.io/kubernetes/pkg/controller" serviceaccountadmission "k8s.io/kubernetes/plugin/pkg/admission/serviceaccount" configapi "github.com/openshift/origin/pkg/cmd/server/apis/config" - "github.com/openshift/origin/pkg/cmd/server/crypto" "github.com/openshift/origin/pkg/cmd/util/variable" ) @@ -56,8 +51,6 @@ func getOpenShiftClientEnvVars(options configapi.MasterConfig) ([]kapi.EnvVar, e // OpenshiftControllerConfig is the runtime (non-serializable) config object used to // launch the set of openshift (not kube) controllers. type OpenshiftControllerConfig struct { - ServiceAccountTokenControllerOptions ServiceAccountTokenControllerOptions - ServiceAccountControllerOptions ServiceAccountControllerOptions BuildControllerConfig BuildControllerConfig @@ -83,8 +76,6 @@ type OpenshiftControllerConfig struct { func (c *OpenshiftControllerConfig) GetControllerInitializers() (map[string]InitFunc, error) { ret := map[string]InitFunc{} - ret["openshift.io/serviceaccount"] = c.ServiceAccountControllerOptions.RunController - ret["openshift.io/serviceaccount-pull-secrets"] = RunServiceAccountPullSecretsController ret["openshift.io/origin-namespace"] = RunOriginNamespaceController ret["openshift.io/service-serving-cert"] = c.ServiceServingCertsControllerOptions.RunController @@ -115,63 +106,10 @@ func (c *OpenshiftControllerConfig) GetControllerInitializers() (map[string]Init return ret, nil } -// NewOpenShiftControllerPreStartInitializers returns list of initializers for controllers -// that needed to be run before any other controller is started. -// Typically this has to done for the serviceaccount-token controller as it provides -// tokens to other controllers. -func (c *OpenshiftControllerConfig) ServiceAccountContentControllerInit() InitFunc { - return c.ServiceAccountTokenControllerOptions.RunController -} - func BuildOpenshiftControllerConfig(options configapi.MasterConfig) (*OpenshiftControllerConfig, error) { var err error ret := &OpenshiftControllerConfig{} - _, loopbackClientConfig, err := configapi.GetInternalKubeClient(options.MasterClients.OpenShiftLoopbackKubeConfig, options.MasterClients.OpenShiftLoopbackClientConnectionOverrides) - if err != nil { - return nil, err - } - - ret.ServiceAccountTokenControllerOptions = ServiceAccountTokenControllerOptions{ - RootClientBuilder: kcontroller.SimpleControllerClientBuilder{ - ClientConfig: loopbackClientConfig, - }, - } - if len(options.ServiceAccountConfig.PrivateKeyFile) > 0 { - ret.ServiceAccountTokenControllerOptions.PrivateKey, err = cert.PrivateKeyFromFile(options.ServiceAccountConfig.PrivateKeyFile) - if err != nil { - return nil, fmt.Errorf("error reading signing key for Service Account Token Manager: %v", err) - } - } - if len(options.ServiceAccountConfig.MasterCA) > 0 { - ret.ServiceAccountTokenControllerOptions.RootCA, err = ioutil.ReadFile(options.ServiceAccountConfig.MasterCA) - if err != nil { - return nil, fmt.Errorf("error reading master ca file for Service Account Token Manager: %s: %v", options.ServiceAccountConfig.MasterCA, err) - } - if _, err := cert.ParseCertsPEM(ret.ServiceAccountTokenControllerOptions.RootCA); err != nil { - return nil, fmt.Errorf("error parsing master ca file for Service Account Token Manager: %s: %v", options.ServiceAccountConfig.MasterCA, err) - } - } - if options.ControllerConfig.ServiceServingCert.Signer != nil && len(options.ControllerConfig.ServiceServingCert.Signer.CertFile) > 0 { - certFile := options.ControllerConfig.ServiceServingCert.Signer.CertFile - serviceServingCA, err := ioutil.ReadFile(certFile) - if err != nil { - return nil, fmt.Errorf("error reading ca file for Service Serving Certificate Signer: %s: %v", certFile, err) - } - if _, err := crypto.CertsFromPEM(serviceServingCA); err != nil { - return nil, fmt.Errorf("error parsing ca file for Service Serving Certificate Signer: %s: %v", certFile, err) - } - - // if we have a rootCA bundle add that too. The rootCA will be used when hitting the default master service, since those are signed - // using a different CA by default. The rootCA's key is more closely guarded than ours and if it is compromised, that power could - // be used to change the trusted signers for every pod anyway, so we're already effectively trusting it. - if len(ret.ServiceAccountTokenControllerOptions.RootCA) > 0 { - ret.ServiceAccountTokenControllerOptions.ServiceServingCA = append(ret.ServiceAccountTokenControllerOptions.ServiceServingCA, ret.ServiceAccountTokenControllerOptions.RootCA...) - ret.ServiceAccountTokenControllerOptions.ServiceServingCA = append(ret.ServiceAccountTokenControllerOptions.ServiceServingCA, []byte("\n")...) - } - ret.ServiceAccountTokenControllerOptions.ServiceServingCA = append(ret.ServiceAccountTokenControllerOptions.ServiceServingCA, serviceServingCA...) - } - ret.ServiceAccountControllerOptions = ServiceAccountControllerOptions{ ManagedNames: options.ServiceAccountConfig.ManagedNames, } diff --git a/pkg/cmd/server/origin/controller/serviceaccount.go b/pkg/cmd/server/origin/controller/serviceaccount.go index 50b7543c88c1..4de7a2e6515c 100644 --- a/pkg/cmd/server/origin/controller/serviceaccount.go +++ b/pkg/cmd/server/origin/controller/serviceaccount.go @@ -4,9 +4,7 @@ import ( "github.com/golang/glog" kapiv1 "k8s.io/api/core/v1" - "k8s.io/kubernetes/pkg/controller" sacontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" - "k8s.io/kubernetes/pkg/serviceaccount" "github.com/openshift/origin/pkg/cmd/server/bootstrappolicy" serviceaccountcontrollers "github.com/openshift/origin/pkg/serviceaccounts/controllers" @@ -49,37 +47,6 @@ func (c *ServiceAccountControllerOptions) RunController(ctx ControllerContext) ( return true, nil } -type ServiceAccountTokenControllerOptions struct { - RootCA []byte - ServiceServingCA []byte - PrivateKey interface{} - - RootClientBuilder controller.SimpleControllerClientBuilder -} - -func (c *ServiceAccountTokenControllerOptions) RunController(ctx ControllerContext) (bool, error) { - if c.PrivateKey == nil { - glog.Infof("Skipped starting Service Account Token Manager, no private key specified") - return false, nil - } - - controller, err := sacontroller.NewTokensController( - ctx.ExternalKubeInformers.Core().V1().ServiceAccounts(), - ctx.ExternalKubeInformers.Core().V1().Secrets(), - c.RootClientBuilder.ClientOrDie(bootstrappolicy.InfraServiceAccountTokensControllerServiceAccountName), - sacontroller.TokensControllerOptions{ - TokenGenerator: serviceaccount.JWTTokenGenerator(c.PrivateKey), - RootCA: c.RootCA, - ServiceServingCA: c.ServiceServingCA, - }, - ) - if err != nil { - return true, nil - } - go controller.Run(int(ctx.OpenshiftControllerOptions.ServiceAccountTokenOptions.ConcurrentSyncs), ctx.Stop) - return true, nil -} - func RunServiceAccountPullSecretsController(ctx ControllerContext) (bool, error) { kc := ctx.ClientBuilder.ClientOrDie(bootstrappolicy.InfraServiceAccountPullSecretsControllerServiceAccountName) diff --git a/pkg/cmd/server/start/start_kube_controller_manager.go b/pkg/cmd/server/start/start_kube_controller_manager.go index e698fb4cdd75..4515254f2a68 100644 --- a/pkg/cmd/server/start/start_kube_controller_manager.go +++ b/pkg/cmd/server/start/start_kube_controller_manager.go @@ -21,8 +21,6 @@ func computeKubeControllerManagerArgs(kubeconfigFile, saPrivateKeyFile, saRootCA "-tokencleaner", // we have to configure this separately until it is generic "-horizontalpodautoscaling", - // we carry patches on this. For now.... - "-serviceaccount-token", } } if _, ok := cmdLineArgs["service-account-private-key-file"]; !ok { diff --git a/pkg/cmd/server/start/start_master.go b/pkg/cmd/server/start/start_master.go index cb908ce3e692..714545db114e 100644 --- a/pkg/cmd/server/start/start_master.go +++ b/pkg/cmd/server/start/start_master.go @@ -6,12 +6,10 @@ import ( "io" "io/ioutil" "net" - "net/http" "os" "path" "path/filepath" "strings" - "time" "github.com/coreos/go-systemd/daemon" "github.com/golang/glog" @@ -21,7 +19,6 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" utilwait "k8s.io/apimachinery/pkg/util/wait" clientgoclientset "k8s.io/client-go/kubernetes" v1core "k8s.io/client-go/kubernetes/typed/core/v1" @@ -431,19 +428,34 @@ func (m *Master) Start() error { go runEmbeddedScheduler(m.config.MasterClients.OpenShiftLoopbackKubeConfig, m.config.KubernetesMasterConfig.SchedulerConfigFile, m.config.KubernetesMasterConfig.SchedulerArguments) go func() { - kubeControllerConfigBytes, err := configapilatest.WriteYAML(m.config) + kubeControllerConfigShallowCopy := *m.config + // this creates using 0700 + kubeControllerConfigDir, err := ioutil.TempDir("", "openshift-kube-controller-manager-config-") if err != nil { glog.Fatal(err) } - // this creates using 0600 - kubeControllerConfigFile, err := ioutil.TempFile("", "openshift-kube-controler-manager-config.yaml") + defer func() { + os.RemoveAll(kubeControllerConfigDir) + }() + if m.config.ControllerConfig.ServiceServingCert.Signer != nil && len(m.config.ControllerConfig.ServiceServingCert.Signer.CertFile) > 0 { + caBytes, err := ioutil.ReadFile(m.config.ControllerConfig.ServiceServingCert.Signer.CertFile) + if err != nil { + glog.Fatal(err) + } + serviceServingCertSignerCAFile := path.Join(kubeControllerConfigDir, "service-signer.crt") + if err := ioutil.WriteFile(serviceServingCertSignerCAFile, caBytes, 0644); err != nil { + glog.Fatal(err) + } + + // we need to tweak the master config file with a relative ref, but to do that we need to copy it + kubeControllerConfigShallowCopy.ControllerConfig.ServiceServingCert.Signer = &configapi.CertInfo{CertFile: "service-signer.crt"} + } + kubeControllerConfigBytes, err := configapilatest.WriteYAML(&kubeControllerConfigShallowCopy) if err != nil { glog.Fatal(err) } - defer func() { - os.Remove(kubeControllerConfigFile.Name()) - }() - if err := ioutil.WriteFile(kubeControllerConfigFile.Name(), kubeControllerConfigBytes, 0644); err != nil { + masterConfigFile := path.Join(kubeControllerConfigDir, "master-config.yaml") + if err := ioutil.WriteFile(masterConfigFile, kubeControllerConfigBytes, 0644); err != nil { glog.Fatal(err) } @@ -452,7 +464,7 @@ func (m *Master) Start() error { m.config.ServiceAccountConfig.PrivateKeyFile, m.config.ServiceAccountConfig.MasterCA, m.config.KubernetesMasterConfig.PodEvictionTimeout, - kubeControllerConfigFile.Name(), + masterConfigFile, m.config.VolumeConfig.DynamicProvisioningEnabled, ) }() @@ -602,40 +614,6 @@ func startControllers(options configapi.MasterConfig, allocationController origi return err } - // We need to start the serviceaccount-tokens controller first as it provides token - // generation for other controllers. - startSATokenController := openshiftControllerConfig.ServiceAccountContentControllerInit() - if enabled, err := startSATokenController(controllerContext); err != nil { - return fmt.Errorf("Error starting serviceaccount-token controller: %v", err) - } else if !enabled { - glog.Warningf("Skipping serviceaccount-token controller") - } else { - glog.Infof("Started serviceaccount-token controller") - } - - // The service account controllers require informers in order to create service account tokens - // for other controllers, which means we need to start their informers (which use the privileged - // loopback client) before the other controllers will run. - controllerContext.ExternalKubeInformers.Start(controllerContext.Stop) - - // right now we have controllers which are relying on the ability to make requests before the bootstrap policy is in place - // In 3.7, we will be fixed by the post start hook that prevents readiness unless policy is in place - // for 3.6, just make sure we don't proceed until the garbage collector can hit discovery - // wait for bootstrap permissions to be established. This check isn't perfect, but it ensures that at least the controllers checking discovery can succeed - gcClientset := controllerContext.ClientBuilder.ClientOrDie("generic-garbage-collector") - err = wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) { - result := gcClientset.Discovery().RESTClient().Get().AbsPath("/apis").Do() - var statusCode int - result.StatusCode(&statusCode) - if statusCode >= http.StatusOK && statusCode < http.StatusMultipleChoices { - return true, nil - } - return false, nil - }) - if err != nil { - return err - } - // the service account passed for the recyclable volume plugins needs to exist. We want to do this via the init function, but its a kube init function // for the rebase, create that service account here // TODO make this a lot cleaner