From 1d2c1a7037ea9c774f44f4081d5ca0ce2ffaee62 Mon Sep 17 00:00:00 2001 From: David Eads Date: Tue, 21 Aug 2018 13:15:47 -0400 Subject: [PATCH] wire hypershift openshift-kube-apiserver to use minimal patching strategy --- hack/import-restrictions.json | 3 +- install/kube-dns/install.yaml | 2 +- install/kube-proxy/install.yaml | 2 + .../openshiftapiserver/openshift_apiserver.go | 27 +- .../openshiftkubeapiserver/flags.go | 271 ++++++++++++++++++ .../openshiftkubeapiserver/patch.go | 85 ++++-- .../patch_handlerchain.go | 18 +- pkg/cmd/openshift-kube-apiserver/server.go | 60 ++-- .../server/origin/admission/chain_builder.go | 12 +- .../server/origin/admission/config_test.go | 52 ++-- pkg/cmd/server/origin/admission/register.go | 4 +- pkg/cmd/server/origin/admission/sync_test.go | 4 +- pkg/cmd/server/origin/master.go | 8 +- test/integration/master_routes_test.go | 1 + 14 files changed, 446 insertions(+), 103 deletions(-) create mode 100644 pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/flags.go diff --git a/hack/import-restrictions.json b/hack/import-restrictions.json index 14327d3d68de..d2ce71d03774 100644 --- a/hack/import-restrictions.json +++ b/hack/import-restrictions.json @@ -50,7 +50,8 @@ "github.com/openshift/origin/pkg/quota/admission/clusterresourceoverride", "github.com/openshift/origin/pkg/oc/cli/admin/createbootstrapprojecttemplate", "github.com/openshift/origin/pkg/cmd/server", - "github.com/openshift/origin/pkg/cmd/openshift-apiserver" + "github.com/openshift/origin/pkg/cmd/openshift-apiserver", + "github.com/openshift/origin/pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver" ], "forbiddenImportPackageRoots": [ "github.com/openshift/origin/pkg/apps/apiserver", diff --git a/install/kube-dns/install.yaml b/install/kube-dns/install.yaml index fc70f828119e..36e89903fc37 100644 --- a/install/kube-dns/install.yaml +++ b/install/kube-dns/install.yaml @@ -42,7 +42,7 @@ objects: spec: serviceAccountName: kube-dns containers: - - name: kube-proxy + - name: kube-dns image: ${IMAGE} imagePullPolicy: ${OPENSHIFT_PULL_POLICY} command: ["openshift", "start", "network"] diff --git a/install/kube-proxy/install.yaml b/install/kube-proxy/install.yaml index c317917fa96b..1ccbcea89ff9 100644 --- a/install/kube-proxy/install.yaml +++ b/install/kube-proxy/install.yaml @@ -30,6 +30,8 @@ objects: - kind: ServiceAccount name: kube-proxy namespace: ${NAMESPACE} + - kind: Group + name: system:nodes roleRef: kind: ClusterRole name: system:node-proxier diff --git a/pkg/cmd/openshift-apiserver/openshiftapiserver/openshift_apiserver.go b/pkg/cmd/openshift-apiserver/openshiftapiserver/openshift_apiserver.go index f186897e755f..cbb4f2fe0314 100644 --- a/pkg/cmd/openshift-apiserver/openshiftapiserver/openshift_apiserver.go +++ b/pkg/cmd/openshift-apiserver/openshiftapiserver/openshift_apiserver.go @@ -26,6 +26,7 @@ import ( kapi "k8s.io/kubernetes/pkg/apis/core" coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" + rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest" rbacregistryvalidation "k8s.io/kubernetes/pkg/registry/rbac/validation" rbacauthorizer "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac" @@ -578,6 +579,16 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) AddOpenshiftVersionRoute(s.GenericAPIServer.Handler.GoRestfulContainer, "/version/openshift") // register our poststarthooks + s.GenericAPIServer.AddPostStartHookOrDie("authorization.openshift.io-bootstrapclusterroles", + func(context genericapiserver.PostStartHookContext) error { + newContext := genericapiserver.PostStartHookContext{ + LoopbackClientConfig: c.ExtraConfig.KubeAPIServerClientConfig, + StopCh: context.StopCh, + } + return bootstrapData(bootstrappolicy.Policy()).EnsureRBACPolicy()(newContext) + + }) + s.GenericAPIServer.AddPostStartHookOrDie("authorization.openshift.io-ensureopenshift-infra", c.EnsureOpenShiftInfraNamespace) s.GenericAPIServer.AddPostStartHookOrDie("project.openshift.io-projectcache", c.startProjectCache) s.GenericAPIServer.AddPostStartHookOrDie("project.openshift.io-projectauthorizationcache", c.startProjectAuthorizationCache) s.GenericAPIServer.AddPostStartHookOrDie("security.openshift.io-bootstrapscc", c.bootstrapSCC) @@ -712,13 +723,13 @@ func (c *completedConfig) bootstrapSCC(context genericapiserver.PostStartHookCon } // EnsureOpenShiftInfraNamespace is called as part of global policy initialization to ensure infra namespace exists -func EnsureOpenShiftInfraNamespace(context genericapiserver.PostStartHookContext) error { +func (c *completedConfig) EnsureOpenShiftInfraNamespace(context genericapiserver.PostStartHookContext) error { namespaceName := bootstrappolicy.DefaultOpenShiftInfraNamespace var coreClient coreclient.CoreInterface err := wait.Poll(1*time.Second, 30*time.Second, func() (bool, error) { var err error - coreClient, err = coreclient.NewForConfig(context.LoopbackClientConfig) + coreClient, err = coreclient.NewForConfig(c.ExtraConfig.KubeAPIServerClientConfig) if err != nil { utilruntime.HandleError(fmt.Errorf("unable to initialize client: %v", err)) return false, nil @@ -744,3 +755,15 @@ func EnsureOpenShiftInfraNamespace(context genericapiserver.PostStartHookContext return nil } + +// bootstrapData casts our policy data to the rbacrest helper that can +// materialize the policy. +func bootstrapData(data *bootstrappolicy.PolicyData) *rbacrest.PolicyData { + return &rbacrest.PolicyData{ + ClusterRoles: data.ClusterRoles, + ClusterRoleBindings: data.ClusterRoleBindings, + Roles: data.Roles, + RoleBindings: data.RoleBindings, + ClusterRolesToAggregate: data.ClusterRolesToAggregate, + } +} diff --git a/pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/flags.go b/pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/flags.go new file mode 100644 index 000000000000..d1fab96d1438 --- /dev/null +++ b/pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/flags.go @@ -0,0 +1,271 @@ +package openshiftkubeapiserver + +import ( + "fmt" + "net" + "sort" + + "io/ioutil" + + "bytes" + + configapi "github.com/openshift/origin/pkg/cmd/server/apis/config" + configapilatest "github.com/openshift/origin/pkg/cmd/server/apis/config/latest" + originadmission "github.com/openshift/origin/pkg/cmd/server/origin/admission" + "k8s.io/apimachinery/pkg/runtime" +) + +func ConfigToFlags(kubeAPIServerConfig *configapi.MasterConfig) ([]string, error) { + args := map[string][]string{} + for key, slice := range kubeAPIServerConfig.KubernetesMasterConfig.APIServerArguments { + for _, val := range slice { + args[key] = append(args[key], val) + } + } + + host, portString, err := net.SplitHostPort(kubeAPIServerConfig.ServingInfo.BindAddress) + if err != nil { + return nil, err + } + + // these flags are overridden by a patch + // admission-control + // authentication-token-webhook-cache-ttl + // authentication-token-webhook-config-file + // authorization-mode + // authorization-policy-file + // authorization-webhook-cache-authorized-ttl + // authorization-webhook-cache-unauthorized-ttl + // authorization-webhook-config-file + // basic-auth-file + // enable-aggregator-routing + // enable-bootstrap-token-auth + // oidc-client-id + // oidc-groups-claim + // oidc-groups-prefix + // oidc-issuer-url + // oidc-required-claim + // oidc-signing-algs + // oidc-username-claim + // oidc-username-prefix + // service-account-lookup + // token-auth-file + + // alsologtostderr - don't know whether to change it + // apiserver-count - ignored, hopefully we don't have to fix via patch + // cert-dir - ignored because we set certs + + // these flags were never supported via config + // cloud-config + // cloud-provider + // cloud-provider-gce-lb-src-cidrs + // contention-profiling + // default-not-ready-toleration-seconds + // default-unreachable-toleration-seconds + // default-watch-cache-size + // delete-collection-workers + // deserialization-cache-size + // enable-garbage-collector + // etcd-compaction-interval + // etcd-count-metric-poll-period + // etcd-servers-overrides + // experimental-encryption-provider-config + // feature-gates + // http2-max-streams-per-connection + // insecure-bind-address + // kubelet-timeout + // log-backtrace-at + // log-dir + // log-flush-frequency + // logtostderr + // master-service-namespace + // max-connection-bytes-per-sec + // profiling + // request-timeout + // runtime-config + // service-account-api-audiences + // service-account-issuer + // service-account-key-file + // service-account-max-token-expiration + // service-account-signing-key-file + // stderrthreshold + // storage-versions + // target-ram-mb + // v + // version + // vmodule + // watch-cache + // watch-cache-sizes + + // TODO, we need to set these in order to enable the right admission plugins in each of the servers + // TODO this is needed for a viable cluster up + admissionFlags, err := admissionFlags(kubeAPIServerConfig) + if err != nil { + return nil, err + } + for flag, value := range admissionFlags { + setIfUnset(args, flag, value...) + } + setIfUnset(args, "allow-privileged", "true") + setIfUnset(args, "anonymous-auth", "false") + setIfUnset(args, "authorization-mode", "RBAC", "Node") // overridden later, but this runs the poststarthook for bootstrapping RBAC + for flag, value := range auditFlags(kubeAPIServerConfig) { + setIfUnset(args, flag, value...) + } + setIfUnset(args, "bind-address", host) + setIfUnset(args, "client-ca-file", kubeAPIServerConfig.ServingInfo.ClientCA) + setIfUnset(args, "cors-allowed-origins", kubeAPIServerConfig.CORSAllowedOrigins...) + setIfUnset(args, "enable-logs-handler", "false") + setIfUnset(args, "enable-swagger-ui", "true") + setIfUnset(args, "endpoint-reconciler-type", "lease") + setIfUnset(args, "etcd-cafile", kubeAPIServerConfig.EtcdClientInfo.CA) + setIfUnset(args, "etcd-certfile", kubeAPIServerConfig.EtcdClientInfo.ClientCert.CertFile) + setIfUnset(args, "etcd-keyfile", kubeAPIServerConfig.EtcdClientInfo.ClientCert.KeyFile) + setIfUnset(args, "etcd-prefix", kubeAPIServerConfig.EtcdStorageConfig.KubernetesStoragePrefix) + setIfUnset(args, "etcd-servers", kubeAPIServerConfig.EtcdClientInfo.URLs...) + setIfUnset(args, "insecure-port", "0") + setIfUnset(args, "kubelet-certificate-authority", kubeAPIServerConfig.KubeletClientInfo.CA) + setIfUnset(args, "kubelet-client-certificate", kubeAPIServerConfig.KubeletClientInfo.ClientCert.CertFile) + setIfUnset(args, "kubelet-client-key", kubeAPIServerConfig.KubeletClientInfo.ClientCert.KeyFile) + setIfUnset(args, "kubelet-https", "true") + setIfUnset(args, "kubelet-preferred-address-types", "Hostname", "InternalIP", "ExternalIP") + setIfUnset(args, "kubelet-read-only-port", "0") + setIfUnset(args, "kubernetes-service-node-port", "0") + setIfUnset(args, "max-mutating-requests-inflight", fmt.Sprintf("%d", kubeAPIServerConfig.ServingInfo.MaxRequestsInFlight/2)) + setIfUnset(args, "max-requests-inflight", fmt.Sprintf("%d", kubeAPIServerConfig.ServingInfo.MaxRequestsInFlight)) + setIfUnset(args, "min-request-timeout", fmt.Sprintf("%d", kubeAPIServerConfig.ServingInfo.RequestTimeoutSeconds)) + setIfUnset(args, "proxy-client-cert-file", kubeAPIServerConfig.AggregatorConfig.ProxyClientInfo.CertFile) + setIfUnset(args, "proxy-client-key-file", kubeAPIServerConfig.AggregatorConfig.ProxyClientInfo.KeyFile) + setIfUnset(args, "requestheader-allowed-names", kubeAPIServerConfig.AuthConfig.RequestHeader.ClientCommonNames...) + setIfUnset(args, "requestheader-client-ca-file", kubeAPIServerConfig.AuthConfig.RequestHeader.ClientCA) + setIfUnset(args, "requestheader-extra-headers-prefix", kubeAPIServerConfig.AuthConfig.RequestHeader.ExtraHeaderPrefixes...) + setIfUnset(args, "requestheader-group-headers", kubeAPIServerConfig.AuthConfig.RequestHeader.GroupHeaders...) + setIfUnset(args, "requestheader-username-headers", kubeAPIServerConfig.AuthConfig.RequestHeader.UsernameHeaders...) + setIfUnset(args, "secure-port", portString) + setIfUnset(args, "service-cluster-ip-range", kubeAPIServerConfig.KubernetesMasterConfig.ServicesSubnet) + setIfUnset(args, "service-node-port-range", kubeAPIServerConfig.KubernetesMasterConfig.ServicesNodePortRange) + setIfUnset(args, "storage-backend", "etcd3") + setIfUnset(args, "storage-media-type", "application/vnd.kubernetes.protobuf") + setIfUnset(args, "tls-cert-file", kubeAPIServerConfig.ServingInfo.ServerCert.CertFile) + setIfUnset(args, "tls-cipher-suites", kubeAPIServerConfig.ServingInfo.CipherSuites...) + setIfUnset(args, "tls-min-version", kubeAPIServerConfig.ServingInfo.MinTLSVersion) + setIfUnset(args, "tls-private-key-file", kubeAPIServerConfig.ServingInfo.ServerCert.KeyFile) + // TODO re-enable SNI for cluster up + // tls-sni-cert-key + setIfUnset(args, "secure-port", portString) + + var keys []string + for key := range args { + keys = append(keys, key) + } + sort.Strings(keys) + + var arguments []string + for _, key := range keys { + for _, token := range args[key] { + arguments = append(arguments, fmt.Sprintf("--%s=%v", key, token)) + } + } + return arguments, nil +} + +// currently for cluster up, audit is just broken. +// TODO fix this +func auditFlags(kubeAPIServerConfig *configapi.MasterConfig) map[string][]string { + args := map[string][]string{} + for key, slice := range kubeAPIServerConfig.KubernetesMasterConfig.APIServerArguments { + for _, val := range slice { + args[key] = append(args[key], val) + } + } + + return args +} + +func setIfUnset(cmdLineArgs map[string][]string, key string, value ...string) { + if _, ok := cmdLineArgs[key]; !ok { + cmdLineArgs[key] = value + } +} + +func admissionFlags(kubeAPIServerConfig *configapi.MasterConfig) (map[string][]string, error) { + args := map[string][]string{} + + forceOn := []string{} + forceOff := []string{} + pluginConfig := map[string]configapi.AdmissionPluginConfig{} + for pluginName, config := range kubeAPIServerConfig.AdmissionConfig.DeepCopy().PluginConfig { + if len(config.Location) > 0 { + content, err := ioutil.ReadFile(config.Location) + if err != nil { + return nil, err + } + // if the config isn't a DefaultAdmissionConfig, then assume we're enabled (we were called after all) + // if the config *is* a DefaultAdmissionConfig and it explicitly said + obj, err := configapilatest.ReadYAML(bytes.NewBuffer(content)) + // if we can't read it, let the plugin deal with it + // if nothing was there, let the plugin deal with it + if err != nil || obj == nil { + forceOn = append(forceOn, pluginName) + config.Location = "" + config.Configuration = &runtime.Unknown{Raw: content} + pluginConfig[pluginName] = *config + continue + } + + if defaultConfig, ok := obj.(*configapi.DefaultAdmissionConfig); !ok { + forceOn = append(forceOn, pluginName) + config.Location = "" + config.Configuration = &runtime.Unknown{Raw: content} + pluginConfig[pluginName] = *config + continue + + } else if defaultConfig.Disable { + forceOff = append(forceOff, pluginName) + continue + } else { + forceOn = append(forceOn, pluginName) + continue + } + + continue + } + // if it wasn't a DefaultAdmissionConfig object, let the plugin deal with it + if defaultConfig, ok := config.Configuration.(*configapi.DefaultAdmissionConfig); !ok { + forceOn = append(forceOn, pluginName) + pluginConfig[pluginName] = *config + continue + + } else if defaultConfig.Disable { + forceOff = append(forceOff, pluginName) + continue + } else { + forceOn = append(forceOn, pluginName) + continue + } + + } + upstreamAdmissionConfig, err := originadmission.ConvertOpenshiftAdmissionConfigToKubeAdmissionConfig(pluginConfig) + if err != nil { + return nil, err + } + configBytes, err := configapilatest.WriteYAML(upstreamAdmissionConfig) + if err != nil { + return nil, err + } + + tempFile, err := ioutil.TempFile("", "kubeapiserver-admission-config.yaml") + if err != nil { + return nil, err + } + if _, err := tempFile.Write(configBytes); err != nil { + return nil, err + } + tempFile.Close() + + setIfUnset(args, "admission-control-config-file", tempFile.Name()) + setIfUnset(args, "disable-admission-plugins", forceOff...) + setIfUnset(args, "enable-admission-plugins", forceOn...) + + return args, nil +} diff --git a/pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/patch.go b/pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/patch.go index a0d46a558e64..a2c0dee3b507 100644 --- a/pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/patch.go +++ b/pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/patch.go @@ -1,23 +1,27 @@ package openshiftkubeapiserver import ( + "github.com/openshift/origin/pkg/admission/namespaceconditions" configapi "github.com/openshift/origin/pkg/cmd/server/apis/config" usercache "github.com/openshift/origin/pkg/user/cache" "k8s.io/apiserver/pkg/admission" + admissionmetrics "k8s.io/apiserver/pkg/admission/metrics" genericapiserver "k8s.io/apiserver/pkg/server" - cacheddiscovery "k8s.io/client-go/discovery/cached" clientgoinformers "k8s.io/client-go/informers" kexternalinformers "k8s.io/client-go/informers" "k8s.io/client-go/rest" - "k8s.io/client-go/restmapper" + "k8s.io/kubernetes/cmd/kube-apiserver/app" internalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" "k8s.io/kubernetes/pkg/master" + "k8s.io/kubernetes/pkg/quota/generic" + "k8s.io/kubernetes/pkg/quota/install" oauthclient "github.com/openshift/client-go/oauth/clientset/versioned" oauthinformer "github.com/openshift/client-go/oauth/informers/externalversions" userclient "github.com/openshift/client-go/user/clientset/versioned" userinformer "github.com/openshift/client-go/user/informers/externalversions" + oadmission "github.com/openshift/origin/pkg/cmd/server/admission" originadmission "github.com/openshift/origin/pkg/cmd/server/origin/admission" imageinformer "github.com/openshift/origin/pkg/image/generated/informers/internalversion" imageclient "github.com/openshift/origin/pkg/image/generated/internalclientset" @@ -30,7 +34,9 @@ import ( "time" "github.com/openshift/origin/pkg/cmd/openshift-apiserver/openshiftapiserver" - "k8s.io/apimachinery/pkg/util/wait" + "github.com/openshift/origin/pkg/cmd/openshift-apiserver/openshiftapiserver/configprocessing" + "github.com/openshift/origin/pkg/image/apiserver/registryhostname" + "k8s.io/apiserver/pkg/server/options" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" ) @@ -38,19 +44,16 @@ import ( type KubeAPIServerServerPatchContext struct { initialized bool - RESTMapper *restmapper.DeferredDiscoveryRESTMapper postStartHooks map[string]genericapiserver.PostStartHookFunc informerStartFuncs []func(stopCh <-chan struct{}) } -type KubeAPIServerConfigFunc func(config *master.Config, internalInformers internalinformers.SharedInformerFactory, kubeInformers clientgoinformers.SharedInformerFactory, pluginInitializers *[]admission.PluginInitializer, stopCh <-chan struct{}) (genericapiserver.DelegationTarget, error) - -func NewOpenShiftKubeAPIServerConfigPatch(delegateAPIServer genericapiserver.DelegationTarget, kubeAPIServerConfig *configapi.MasterConfig) (KubeAPIServerConfigFunc, *KubeAPIServerServerPatchContext) { +func NewOpenShiftKubeAPIServerConfigPatch(delegateAPIServer genericapiserver.DelegationTarget, kubeAPIServerConfig *configapi.MasterConfig) (app.KubeAPIServerConfigFunc, *KubeAPIServerServerPatchContext) { patchContext := &KubeAPIServerServerPatchContext{ postStartHooks: map[string]genericapiserver.PostStartHookFunc{}, } - return func(config *master.Config, internalInformers internalinformers.SharedInformerFactory, kubeInformers clientgoinformers.SharedInformerFactory, pluginInitializers *[]admission.PluginInitializer, stopCh <-chan struct{}) (genericapiserver.DelegationTarget, error) { - kubeAPIServerInformers, err := NewInformers(internalInformers, kubeInformers, config.GenericConfig.LoopbackClientConfig) + return func(genericConfig *genericapiserver.Config, internalInformers internalinformers.SharedInformerFactory, kubeInformers clientgoinformers.SharedInformerFactory, pluginInitializers *[]admission.PluginInitializer) (genericapiserver.DelegationTarget, error) { + kubeAPIServerInformers, err := NewInformers(internalInformers, kubeInformers, genericConfig.LoopbackClientConfig) if err != nil { return nil, err } @@ -58,44 +61,76 @@ func NewOpenShiftKubeAPIServerConfigPatch(delegateAPIServer genericapiserver.Del // AUTHENTICATOR authenticator, postStartHooks, err := NewAuthenticator( *kubeAPIServerConfig, - config.GenericConfig.LoopbackClientConfig, + genericConfig.LoopbackClientConfig, kubeAPIServerInformers.OpenshiftOAuthInformers.Oauth().V1().OAuthClients().Lister(), kubeAPIServerInformers.OpenshiftUserInformers.User().V1().Groups()) if err != nil { return nil, err } - config.GenericConfig.Authentication.Authenticator = authenticator + genericConfig.Authentication.Authenticator = authenticator for key, fn := range postStartHooks { patchContext.postStartHooks[key] = fn } // END AUTHENTICATOR // AUTHORIZER + genericConfig.RequestInfoResolver = configprocessing.OpenshiftRequestInfoResolver() authorizer := NewAuthorizer(internalInformers, kubeInformers) - config.GenericConfig.Authorization.Authorizer = authorizer + genericConfig.Authorization.Authorizer = authorizer // END AUTHORIZER // ADMISSION - projectCache, err := openshiftapiserver.NewProjectCache(kubeAPIServerInformers.InternalKubernetesInformers.Core().InternalVersion().Namespaces(), config.GenericConfig.LoopbackClientConfig, kubeAPIServerConfig.ProjectConfig.DefaultNodeSelector) + projectCache, err := openshiftapiserver.NewProjectCache(kubeAPIServerInformers.InternalKubernetesInformers.Core().InternalVersion().Namespaces(), genericConfig.LoopbackClientConfig, kubeAPIServerConfig.ProjectConfig.DefaultNodeSelector) if err != nil { return nil, err } clusterQuotaMappingController := openshiftapiserver.NewClusterQuotaMappingController(kubeAPIServerInformers.InternalKubernetesInformers.Core().InternalVersion().Namespaces(), kubeAPIServerInformers.InternalOpenshiftQuotaInformers.Quota().InternalVersion().ClusterResourceQuotas()) - kubeClient, err := kubernetes.NewForConfig(config.GenericConfig.LoopbackClientConfig) + patchContext.postStartHooks["quota.openshift.io-clusterquotamapping"] = func(context genericapiserver.PostStartHookContext) error { + go clusterQuotaMappingController.Run(5, context.StopCh) + return nil + } + kubeClient, err := kubernetes.NewForConfig(genericConfig.LoopbackClientConfig) + if err != nil { + return nil, err + } + registryHostnameRetriever, err := registryhostname.DefaultRegistryHostnameRetriever(genericConfig.LoopbackClientConfig, kubeAPIServerConfig.ImagePolicyConfig.ExternalRegistryHostname, kubeAPIServerConfig.ImagePolicyConfig.InternalRegistryHostname) if err != nil { return nil, err } - discoveryClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery()) - restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) - admissionInitializer, err := originadmission.NewPluginInitializer(*kubeAPIServerConfig, config.GenericConfig.LoopbackClientConfig, kubeAPIServerInformers, config.GenericConfig.Authorization.Authorizer, projectCache, restMapper, clusterQuotaMappingController) + // TODO make a union registry + quotaRegistry := generic.NewRegistry(install.NewQuotaConfigurationForAdmission().Evaluators()) + openshiftPluginInitializer := &oadmission.PluginInitializer{ + ProjectCache: projectCache, + OriginQuotaRegistry: quotaRegistry, + JenkinsPipelineConfig: kubeAPIServerConfig.JenkinsPipelineConfig, + RESTClientConfig: *genericConfig.LoopbackClientConfig, + ClusterResourceQuotaInformer: kubeAPIServerInformers.GetInternalOpenshiftQuotaInformers().Quota().InternalVersion().ClusterResourceQuotas(), + ClusterQuotaMapper: clusterQuotaMappingController.GetClusterQuotaMapper(), + RegistryHostnameRetriever: registryHostnameRetriever, + SecurityInformers: kubeAPIServerInformers.GetInternalOpenshiftSecurityInformers(), + UserInformers: kubeAPIServerInformers.GetOpenshiftUserInformers(), + } if err != nil { return nil, err } - *pluginInitializers = []admission.PluginInitializer{admissionInitializer} + *pluginInitializers = append(*pluginInitializers, openshiftPluginInitializer) + + // set up the decorators we need + namespaceLabelDecorator := namespaceconditions.NamespaceLabelConditions{ + NamespaceClient: kubeClient.CoreV1(), + NamespaceLister: kubeInformers.Core().V1().Namespaces().Lister(), + + SkipLevelZeroNames: originadmission.SkipRunLevelZeroPlugins, + SkipLevelOneNames: originadmission.SkipRunLevelOnePlugins, + } + options.AdmissionDecorator = admission.Decorators{ + admission.DecoratorFunc(namespaceLabelDecorator.WithNamespaceLabelConditions), + admission.DecoratorFunc(admissionmetrics.WithControllerMetrics), + } // END ADMISSION // HANDLER CHAIN (with oauth server and web console) - config.GenericConfig.BuildHandlerChainFunc, postStartHooks, err = BuildHandlerChain(config.GenericConfig, kubeInformers, kubeAPIServerConfig, stopCh) + genericConfig.BuildHandlerChainFunc, postStartHooks, err = BuildHandlerChain(genericConfig, kubeInformers, kubeAPIServerConfig) if err != nil { return nil, err } @@ -105,7 +140,7 @@ func NewOpenShiftKubeAPIServerConfigPatch(delegateAPIServer genericapiserver.Del // END HANDLER CHAIN // CONSTRUCT DELEGATE - nonAPIServerConfig, err := NewOpenshiftNonAPIConfig(config.GenericConfig, kubeInformers, kubeAPIServerConfig) + nonAPIServerConfig, err := NewOpenshiftNonAPIConfig(genericConfig, kubeInformers, kubeAPIServerConfig) if err != nil { return nil, err } @@ -116,7 +151,6 @@ func NewOpenShiftKubeAPIServerConfigPatch(delegateAPIServer genericapiserver.Del // END CONSTRUCT DELEGATE patchContext.informerStartFuncs = append(patchContext.informerStartFuncs, kubeAPIServerInformers.Start) - patchContext.RESTMapper = restMapper patchContext.initialized = true return openshiftNonAPIServer.GenericAPIServer, nil @@ -137,15 +171,6 @@ func (c *KubeAPIServerServerPatchContext) PatchServer(server *master.Master) err } return nil }) - server.GenericAPIServer.AddPostStartHookOrDie("openshift.io-restmapperupdater", func(context genericapiserver.PostStartHookContext) error { - c.RESTMapper.Reset() - go func() { - wait.Until(func() { - c.RESTMapper.Reset() - }, 10*time.Second, context.StopCh) - }() - return nil - }) return nil } diff --git a/pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/patch_handlerchain.go b/pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/patch_handlerchain.go index f783ce8262cd..0a618de9dde0 100644 --- a/pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/patch_handlerchain.go +++ b/pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/patch_handlerchain.go @@ -21,20 +21,28 @@ const ( openShiftOAuthCallbackPrefix = "/oauth2callback" ) -func BuildHandlerChain(genericConfig *genericapiserver.Config, kubeInformers informers.SharedInformerFactory, kubeAPIServerConfig *configapi.MasterConfig, stopCh <-chan struct{}) (func(apiHandler http.Handler, kc *genericapiserver.Config) http.Handler, map[string]genericapiserver.PostStartHookFunc, error) { - webconsoleProxyHandler, err := newWebConsoleProxy(genericConfig, kubeInformers, kubeAPIServerConfig) +func BuildHandlerChain(genericConfig *genericapiserver.Config, kubeInformers informers.SharedInformerFactory, kubeAPIServerConfig *configapi.MasterConfig) (func(apiHandler http.Handler, kc *genericapiserver.Config) http.Handler, map[string]genericapiserver.PostStartHookFunc, error) { + extraPostStartHooks := map[string]genericapiserver.PostStartHookFunc{} + + webconsoleProxyHandler, err := newWebConsoleProxy(kubeInformers, kubeAPIServerConfig) if err != nil { return nil, nil, err } - oauthServerHandler, extraPostStartHooks, err := NewOAuthServerHandler(genericConfig, kubeAPIServerConfig) + oauthServerHandler, newPostStartHooks, err := NewOAuthServerHandler(genericConfig, kubeAPIServerConfig) if err != nil { return nil, nil, err } + for name, fn := range newPostStartHooks { + extraPostStartHooks[name] = fn + } return func(apiHandler http.Handler, genericConfig *genericapiserver.Config) http.Handler { // Machinery that let's use discover the Web Console Public URL accessor := newWebConsolePublicURLAccessor(genericConfig.LoopbackClientConfig) - go accessor.Run(stopCh) + extraPostStartHooks["openshift.io-webconsolepublicurl"] = func(context genericapiserver.PostStartHookContext) error { + go accessor.Run(context.StopCh) + return nil + } // these are after the kube handler handler := versionSkewFilter(apiHandler, kubeAPIServerConfig) @@ -60,7 +68,7 @@ func BuildHandlerChain(genericConfig *genericapiserver.Config, kubeInformers inf nil } -func newWebConsoleProxy(genericConfig *genericapiserver.Config, kubeInformers informers.SharedInformerFactory, kubeAPIServerConfig *configapi.MasterConfig) (http.Handler, error) { +func newWebConsoleProxy(kubeInformers informers.SharedInformerFactory, kubeAPIServerConfig *configapi.MasterConfig) (http.Handler, error) { caBundle, err := ioutil.ReadFile(kubeAPIServerConfig.ControllerConfig.ServiceServingCert.Signer.CertFile) if err != nil { return nil, err diff --git a/pkg/cmd/openshift-kube-apiserver/server.go b/pkg/cmd/openshift-kube-apiserver/server.go index c7bea9b0e7e5..93f581e3e47b 100644 --- a/pkg/cmd/openshift-kube-apiserver/server.go +++ b/pkg/cmd/openshift-kube-apiserver/server.go @@ -1,20 +1,25 @@ package openshift_kube_apiserver import ( + "fmt" + "github.com/golang/glog" kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/sets" utilwait "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/pkg/version" - aggregatorinstall "k8s.io/kube-aggregator/pkg/apis/apiregistration/install" - "k8s.io/kubernetes/pkg/api/legacyscheme" + "k8s.io/apiserver/pkg/admission" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/pkg/capabilities" kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" + "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy" + "github.com/openshift/origin/pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver" configapi "github.com/openshift/origin/pkg/cmd/server/apis/config" "github.com/openshift/origin/pkg/cmd/server/apis/config/validation" - "github.com/openshift/origin/pkg/cmd/server/origin" - "github.com/openshift/origin/pkg/cmd/util/variable" + originadmission "github.com/openshift/origin/pkg/cmd/server/origin/admission" + "k8s.io/kubernetes/pkg/kubeapiserver/options" ) func RunOpenShiftKubeAPIServerServer(masterConfig *configapi.MasterConfig) error { @@ -28,11 +33,6 @@ func RunOpenShiftKubeAPIServerServer(masterConfig *configapi.MasterConfig) error }, }) - // install aggregator types into the scheme so that "normal" RESTOptionsGetters can work for us. - // done in Start() prior to doing any other initialization so we don't mutate the scheme after it is being used by clients in other goroutines. - // TODO: make scheme threadsafe and do this as part of aggregator config building - aggregatorinstall.Install(legacyscheme.Scheme) - validationResults := validation.ValidateMasterConfig(masterConfig, nil) if len(validationResults.Warnings) != 0 { for _, warning := range validationResults.Warnings { @@ -43,22 +43,38 @@ func RunOpenShiftKubeAPIServerServer(masterConfig *configapi.MasterConfig) error return kerrors.NewInvalid(configapi.Kind("MasterConfig"), "master-config.yaml", validationResults.Errors) } - informers := origin.InformerAccess(nil) // use real kube-apiserver loopback client with secret token instead of that from masterConfig.MasterClients.OpenShiftLoopbackKubeConfig - openshiftConfig, err := origin.BuildMasterConfig(*masterConfig, informers) - if err != nil { - return err + bootstrappolicy.ClusterRoles = bootstrappolicy.OpenshiftClusterRoles + bootstrappolicy.ClusterRoleBindings = bootstrappolicy.OpenshiftClusterRoleBindings + + options.AllOrderedPlugins = originadmission.CombinedAdmissionControlPlugins + kubeRegisterAdmission := options.RegisterAllAdmissionPlugins + options.RegisterAllAdmissionPlugins = func(plugins *admission.Plugins) { + kubeRegisterAdmission(plugins) + originadmission.RegisterOpenshiftAdmissionPlugins(plugins) + } + kubeDefaultOffAdmission := options.DefaultOffAdmissionPlugins + options.DefaultOffAdmissionPlugins = func() sets.String { + kubeOff := kubeDefaultOffAdmission() + kubeOff.Delete(originadmission.DefaultOnPlugins.List()...) + return kubeOff } - glog.Infof("Starting master on %s (%s)", masterConfig.ServingInfo.BindAddress, version.Get().String()) - glog.Infof("Public master address is %s", masterConfig.MasterPublicURL) - imageTemplate := variable.NewDefaultImageTemplate() - imageTemplate.Format = masterConfig.ImageConfig.Format - imageTemplate.Latest = masterConfig.ImageConfig.Latest - glog.Infof("Using images from %q", imageTemplate.ExpandOrDie("")) + configPatchFn, serverPatchContext := openshiftkubeapiserver.NewOpenShiftKubeAPIServerConfigPatch(genericapiserver.NewEmptyDelegate(), masterConfig) + app.OpenShiftKubeAPIServerConfigPatch = configPatchFn + app.OpenShiftKubeAPIServerServerPatch = serverPatchContext.PatchServer - if err := openshiftConfig.RunKubeAPIServer(utilwait.NeverStop); err != nil { + cmd := app.NewAPIServerCommand(utilwait.NeverStop) + args, err := openshiftkubeapiserver.ConfigToFlags(masterConfig) + if err != nil { + return err + } + if err := cmd.ParseFlags(args); err != nil { + return err + } + glog.Infof("`kube-apiserver %v`", args) + if err := cmd.RunE(cmd, nil); err != nil { return err } - return nil + return fmt.Errorf("`kube-apiserver %v` exited", args) } diff --git a/pkg/cmd/server/origin/admission/chain_builder.go b/pkg/cmd/server/origin/admission/chain_builder.go index 18fb4f0a90b2..44dc5731f3a5 100644 --- a/pkg/cmd/server/origin/admission/chain_builder.go +++ b/pkg/cmd/server/origin/admission/chain_builder.go @@ -61,8 +61,8 @@ var ( "ResourceQuota", } - // kubeAdmissionPlugins gives the in-order default admission chain for kube resources. - kubeAdmissionPlugins = []string{ + // KubeAdmissionPlugins gives the in-order default admission chain for kube resources. + KubeAdmissionPlugins = []string{ "AlwaysAdmit", "NamespaceAutoProvision", "NamespaceExists", @@ -112,7 +112,7 @@ var ( // combinedAdmissionControlPlugins gives the in-order default admission chain for all resources resources. // When possible, this list is used. The set of openshift+kube chains must exactly match this set. In addition, // the order specified in the openshift and kube chains must match the order here. - combinedAdmissionControlPlugins = []string{ + CombinedAdmissionControlPlugins = []string{ "AlwaysAdmit", "NamespaceAutoProvision", "NamespaceExists", @@ -187,7 +187,7 @@ func NewAdmissionChains( for pluginName, config := range options.AdmissionConfig.PluginConfig { pluginConfig[pluginName] = *config } - upstreamAdmissionConfig, err := convertOpenshiftAdmissionConfigToKubeAdmissionConfig(pluginConfig) + upstreamAdmissionConfig, err := ConvertOpenshiftAdmissionConfigToKubeAdmissionConfig(pluginConfig) if err != nil { return nil, err } @@ -208,7 +208,7 @@ func NewAdmissionChains( admissionPluginConfigFilename = tempFile.Name() } - admissionPluginNames := combinedAdmissionControlPlugins + admissionPluginNames := CombinedAdmissionControlPlugins if len(options.AdmissionConfig.PluginOrderOverride) > 0 { admissionPluginNames = options.AdmissionConfig.PluginOrderOverride } @@ -334,7 +334,7 @@ func splitStream(config io.Reader) (io.Reader, io.Reader, error) { return bytes.NewBuffer(configBytes), bytes.NewBuffer(configBytes), nil } -func convertOpenshiftAdmissionConfigToKubeAdmissionConfig(in map[string]configapi.AdmissionPluginConfig) (*apiserver.AdmissionConfiguration, error) { +func ConvertOpenshiftAdmissionConfigToKubeAdmissionConfig(in map[string]configapi.AdmissionPluginConfig) (*apiserver.AdmissionConfiguration, error) { ret := &apiserver.AdmissionConfiguration{} for _, pluginName := range sets.StringKeySet(in).List() { diff --git a/pkg/cmd/server/origin/admission/config_test.go b/pkg/cmd/server/origin/admission/config_test.go index e11e1bf4da10..82064ba764bd 100644 --- a/pkg/cmd/server/origin/admission/config_test.go +++ b/pkg/cmd/server/origin/admission/config_test.go @@ -21,8 +21,8 @@ import ( // the order of default kube must follow the order of default combined func TestAdmissionPluginChains(t *testing.T) { individualSet := sets.NewString(openshiftAdmissionControlPlugins...) - individualSet.Insert(kubeAdmissionPlugins...) - combinedSet := sets.NewString(combinedAdmissionControlPlugins...) + individualSet.Insert(KubeAdmissionPlugins...) + combinedSet := sets.NewString(CombinedAdmissionControlPlugins...) if !individualSet.Equal(combinedSet) { t.Fatalf("individualSets are missing: %v combinedSet is missing: %v", combinedSet.Difference(individualSet), individualSet.Difference(combinedSet)) @@ -30,26 +30,26 @@ func TestAdmissionPluginChains(t *testing.T) { lastCurrIndex := -1 for _, plugin := range openshiftAdmissionControlPlugins { - for lastCurrIndex = lastCurrIndex + 1; lastCurrIndex < len(combinedAdmissionControlPlugins); lastCurrIndex++ { - if combinedAdmissionControlPlugins[lastCurrIndex] == plugin { + for lastCurrIndex = lastCurrIndex + 1; lastCurrIndex < len(CombinedAdmissionControlPlugins); lastCurrIndex++ { + if CombinedAdmissionControlPlugins[lastCurrIndex] == plugin { break } } - if lastCurrIndex >= len(combinedAdmissionControlPlugins) { + if lastCurrIndex >= len(CombinedAdmissionControlPlugins) { t.Errorf("openshift admission plugins are out of order compared to the combined list. Failed at %v", plugin) } } lastCurrIndex = -1 - for _, plugin := range kubeAdmissionPlugins { - for lastCurrIndex = lastCurrIndex + 1; lastCurrIndex < len(combinedAdmissionControlPlugins); lastCurrIndex++ { - if combinedAdmissionControlPlugins[lastCurrIndex] == plugin { + for _, plugin := range KubeAdmissionPlugins { + for lastCurrIndex = lastCurrIndex + 1; lastCurrIndex < len(CombinedAdmissionControlPlugins); lastCurrIndex++ { + if CombinedAdmissionControlPlugins[lastCurrIndex] == plugin { break } } - if lastCurrIndex >= len(combinedAdmissionControlPlugins) { + if lastCurrIndex >= len(CombinedAdmissionControlPlugins) { t.Errorf("kube admission plugins are out of order compared to the combined list. Failed at %v", plugin) } } @@ -73,7 +73,7 @@ var legacyOpenshiftAdmissionPlugins = sets.NewString( // TestAdmissionPluginNames makes sure that openshift admission plugins are prefixed with `openshift.io/`. func TestAdmissionPluginNames(t *testing.T) { originAdmissionPlugins := admission.NewPlugins() - registerOpenshiftAdmissionPlugins(originAdmissionPlugins) + RegisterOpenshiftAdmissionPlugins(originAdmissionPlugins) for _, plugin := range originAdmissionPlugins.Registered() { if !strings.HasPrefix(plugin, "openshift.io/") && !legacyOpenshiftAdmissionPlugins.Has(plugin) { @@ -84,7 +84,7 @@ func TestAdmissionPluginNames(t *testing.T) { func TestUnusuedKubeAdmissionPlugins(t *testing.T) { allAdmissionPlugins := sets.NewString(OriginAdmissionPlugins.Registered()...) - knownAdmissionPlugins := sets.NewString(combinedAdmissionControlPlugins...) + knownAdmissionPlugins := sets.NewString(CombinedAdmissionControlPlugins...) if unorderedPlugins := allAdmissionPlugins.Difference(knownAdmissionPlugins); len(unorderedPlugins) != 0 { t.Errorf("%v need to be ordered and enabled/disabled", unorderedPlugins.List()) @@ -103,8 +103,8 @@ func TestSeparateAdmissionChainDetection(t *testing.T) { KubernetesMasterConfig: configapi.KubernetesMasterConfig{}, }, admissionChainBuilder: func(pluginNames []string, admissionConfigFilename string, options configapi.MasterConfig, pluginInitializer admission.PluginInitializer, decorator admission.Decorator) (admission.Interface, error) { - if !reflect.DeepEqual(pluginNames, combinedAdmissionControlPlugins) { - t.Errorf("%s: expected %v, got %v", "stock everything", combinedAdmissionControlPlugins, pluginNames) + if !reflect.DeepEqual(pluginNames, CombinedAdmissionControlPlugins) { + t.Errorf("%s: expected %v, got %v", "stock everything", CombinedAdmissionControlPlugins, pluginNames) } return nil, nil }, @@ -118,13 +118,13 @@ func TestSeparateAdmissionChainDetection(t *testing.T) { }, }, admissionChainBuilder: func(pluginNames []string, admissionConfigFilename string, options configapi.MasterConfig, pluginInitializer admission.PluginInitializer, decorator admission.Decorator) (admission.Interface, error) { - isKube := reflect.DeepEqual(pluginNames, combinedAdmissionControlPlugins) + isKube := reflect.DeepEqual(pluginNames, CombinedAdmissionControlPlugins) expectedOrigin := []string{"foo"} isOrigin := reflect.DeepEqual(pluginNames, expectedOrigin) if !isKube && !isOrigin { - t.Errorf("%s: expected either %v or %v, got %v", "specified origin admission order", kubeAdmissionPlugins, expectedOrigin, pluginNames) + t.Errorf("%s: expected either %v or %v, got %v", "specified origin admission order", KubeAdmissionPlugins, expectedOrigin, pluginNames) } return nil, nil @@ -140,10 +140,10 @@ func TestSeparateAdmissionChainDetection(t *testing.T) { }, }, admissionChainBuilder: func(pluginNames []string, admissionConfigFilename string, options configapi.MasterConfig, pluginInitializer admission.PluginInitializer, decorator admission.Decorator) (admission.Interface, error) { - isKube := reflect.DeepEqual(pluginNames, combinedAdmissionControlPlugins) - isOrigin := reflect.DeepEqual(pluginNames, combinedAdmissionControlPlugins) + isKube := reflect.DeepEqual(pluginNames, CombinedAdmissionControlPlugins) + isOrigin := reflect.DeepEqual(pluginNames, CombinedAdmissionControlPlugins) if !isKube && !isOrigin { - t.Errorf("%s: expected either %v or %v, got %v", "specified conflicting plugin configs 01", kubeAdmissionPlugins, openshiftAdmissionControlPlugins, pluginNames) + t.Errorf("%s: expected either %v or %v, got %v", "specified conflicting plugin configs 01", KubeAdmissionPlugins, openshiftAdmissionControlPlugins, pluginNames) } return nil, nil }, @@ -161,8 +161,8 @@ func TestSeparateAdmissionChainDetection(t *testing.T) { }, }, admissionChainBuilder: func(pluginNames []string, admissionConfigFilename string, options configapi.MasterConfig, pluginInitializer admission.PluginInitializer, decorator admission.Decorator) (admission.Interface, error) { - if !reflect.DeepEqual(pluginNames, combinedAdmissionControlPlugins) { - t.Errorf("%s: expected %v, got %v", "specified, non-conflicting plugin configs 01", combinedAdmissionControlPlugins, pluginNames) + if !reflect.DeepEqual(pluginNames, CombinedAdmissionControlPlugins) { + t.Errorf("%s: expected %v, got %v", "specified, non-conflicting plugin configs 01", CombinedAdmissionControlPlugins, pluginNames) } return nil, nil }, @@ -176,25 +176,25 @@ func TestSeparateAdmissionChainDetection(t *testing.T) { } func TestQuotaAdmissionPluginsAreLast(t *testing.T) { - kubeLen := len(kubeAdmissionPlugins) + kubeLen := len(KubeAdmissionPlugins) if kubeLen < 2 { t.Fatalf("must have at least the 2 quota plugins") } - if kubeAdmissionPlugins[kubeLen-2] != "ResourceQuota" { + if KubeAdmissionPlugins[kubeLen-2] != "ResourceQuota" { t.Errorf("kubeAdmissionPlugins must have %s as the next to last plugin", "ResourceQuota") } - if kubeAdmissionPlugins[kubeLen-1] != "openshift.io/ClusterResourceQuota" { + if KubeAdmissionPlugins[kubeLen-1] != "openshift.io/ClusterResourceQuota" { t.Errorf("kubeAdmissionPlugins must have ClusterResourceQuota as the last plugin") } - combinedLen := len(combinedAdmissionControlPlugins) - if combinedAdmissionControlPlugins[combinedLen-2] != "ResourceQuota" { + combinedLen := len(CombinedAdmissionControlPlugins) + if CombinedAdmissionControlPlugins[combinedLen-2] != "ResourceQuota" { t.Errorf("combinedAdmissionControlPlugins must have %s as the next to last plugin", "ResourceQuota") } - if combinedAdmissionControlPlugins[combinedLen-1] != "openshift.io/ClusterResourceQuota" { + if CombinedAdmissionControlPlugins[combinedLen-1] != "openshift.io/ClusterResourceQuota" { t.Errorf("combinedAdmissionControlPlugins must have ClusterResourceQuota as the last plugin") } } diff --git a/pkg/cmd/server/origin/admission/register.go b/pkg/cmd/server/origin/admission/register.go index 63123e4789f2..ede986d21bb9 100644 --- a/pkg/cmd/server/origin/admission/register.go +++ b/pkg/cmd/server/origin/admission/register.go @@ -50,10 +50,10 @@ func init() { func RegisterAllAdmissionPlugins(plugins *admission.Plugins) { kubeapiserver.RegisterAllAdmissionPlugins(plugins) genericapiserver.RegisterAllAdmissionPlugins(plugins) - registerOpenshiftAdmissionPlugins(plugins) + RegisterOpenshiftAdmissionPlugins(plugins) } -func registerOpenshiftAdmissionPlugins(plugins *admission.Plugins) { +func RegisterOpenshiftAdmissionPlugins(plugins *admission.Plugins) { authorizationrestrictusers.Register(plugins) buildjenkinsbootstrapper.Register(plugins) buildsecretinjector.Register(plugins) diff --git a/pkg/cmd/server/origin/admission/sync_test.go b/pkg/cmd/server/origin/admission/sync_test.go index 3780f7720590..27acc3c07d73 100644 --- a/pkg/cmd/server/origin/admission/sync_test.go +++ b/pkg/cmd/server/origin/admission/sync_test.go @@ -44,7 +44,7 @@ func TestKubeAdmissionControllerUsage(t *testing.T) { kubeapiserver.RegisterAllAdmissionPlugins(admission.NewPlugins()) registeredKubePlugins := sets.NewString(OriginAdmissionPlugins.Registered()...) - usedAdmissionPlugins := sets.NewString(kubeAdmissionPlugins...) + usedAdmissionPlugins := sets.NewString(KubeAdmissionPlugins...) if missingPlugins := usedAdmissionPlugins.Difference(registeredKubePlugins); len(missingPlugins) != 0 { t.Errorf("%v not found", missingPlugins.List()) @@ -60,7 +60,7 @@ func TestKubeAdmissionControllerUsage(t *testing.T) { } func TestAdmissionOnOffCoverage(t *testing.T) { - configuredAdmissionPlugins := sets.NewString(combinedAdmissionControlPlugins...) + configuredAdmissionPlugins := sets.NewString(CombinedAdmissionControlPlugins...) allCoveredAdmissionPlugins := sets.String{} allCoveredAdmissionPlugins.Insert(DefaultOnPlugins.List()...) allCoveredAdmissionPlugins.Insert(DefaultOffPlugins.List()...) diff --git a/pkg/cmd/server/origin/master.go b/pkg/cmd/server/origin/master.go index c10559e2210d..38d04e54a253 100644 --- a/pkg/cmd/server/origin/master.go +++ b/pkg/cmd/server/origin/master.go @@ -176,7 +176,7 @@ func (c *MasterConfig) Run(stopCh <-chan struct{}) error { var delegateAPIServer apiserver.DelegationTarget var extraPostStartHooks map[string]apiserver.PostStartHookFunc - c.kubeAPIServerConfig.GenericConfig.BuildHandlerChainFunc, extraPostStartHooks, err = openshiftkubeapiserver.BuildHandlerChain(c.kubeAPIServerConfig.GenericConfig, c.ClientGoKubeInformers, &c.Options, stopCh) + c.kubeAPIServerConfig.GenericConfig.BuildHandlerChainFunc, extraPostStartHooks, err = openshiftkubeapiserver.BuildHandlerChain(c.kubeAPIServerConfig.GenericConfig, c.ClientGoKubeInformers, &c.Options) if err != nil { return err } @@ -225,9 +225,6 @@ func (c *MasterConfig) Run(stopCh <-chan struct{}) error { } // add post-start hooks - aggregatedAPIServer.GenericAPIServer.AddPostStartHookOrDie("authorization.openshift.io-bootstrapclusterroles", bootstrapData(bootstrappolicy.Policy()).EnsureRBACPolicy()) - aggregatedAPIServer.GenericAPIServer.AddPostStartHookOrDie("authorization.openshift.io-ensureopenshift-infra", openshiftapiserver.EnsureOpenShiftInfraNamespace) - for name, fn := range c.additionalPostStartHooks { aggregatedAPIServer.GenericAPIServer.AddPostStartHookOrDie(name, fn) } @@ -247,7 +244,7 @@ func (c *MasterConfig) RunKubeAPIServer(stopCh <-chan struct{}) error { var delegateAPIServer apiserver.DelegationTarget var extraPostStartHooks map[string]apiserver.PostStartHookFunc - c.kubeAPIServerConfig.GenericConfig.BuildHandlerChainFunc, extraPostStartHooks, err = openshiftkubeapiserver.BuildHandlerChain(c.kubeAPIServerConfig.GenericConfig, c.ClientGoKubeInformers, &c.Options, stopCh) + c.kubeAPIServerConfig.GenericConfig.BuildHandlerChainFunc, extraPostStartHooks, err = openshiftkubeapiserver.BuildHandlerChain(c.kubeAPIServerConfig.GenericConfig, c.ClientGoKubeInformers, &c.Options) if err != nil { return err } @@ -284,7 +281,6 @@ func (c *MasterConfig) RunKubeAPIServer(stopCh <-chan struct{}) error { } aggregatedAPIServer.GenericAPIServer.AddPostStartHookOrDie("authorization.openshift.io-bootstrapclusterroles", bootstrapData(bootstrappolicy.Policy()).EnsureRBACPolicy()) - aggregatedAPIServer.GenericAPIServer.AddPostStartHookOrDie("authorization.openshift.io-ensureopenshift-infra", openshiftapiserver.EnsureOpenShiftInfraNamespace) aggregatedAPIServer.GenericAPIServer.AddPostStartHookOrDie("openshift.io-startinformers", func(context apiserver.PostStartHookContext) error { c.InformerStart(context.StopCh) return nil diff --git a/test/integration/master_routes_test.go b/test/integration/master_routes_test.go index f940c92cf808..420812fba4db 100644 --- a/test/integration/master_routes_test.go +++ b/test/integration/master_routes_test.go @@ -122,6 +122,7 @@ var expectedIndex = []string{ "/healthz/poststarthook/oauth.openshift.io-startoauthclientsbootstrapping", "/healthz/poststarthook/openshift.io-restmapperupdater", "/healthz/poststarthook/openshift.io-startinformers", + "/healthz/poststarthook/openshift.io-webconsolepublicurl", "/healthz/poststarthook/project.openshift.io-projectauthorizationcache", "/healthz/poststarthook/project.openshift.io-projectcache", "/healthz/poststarthook/quota.openshift.io-clusterquotamapping",