add kube-apiserver wiring

pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/flags.go

@@ -0,0 +1,177 @@
+package openshiftkubeapiserver
+
+import (
+	"fmt"
+	"net"
+	"sort"
+
+	configapi "github.com/openshift/origin/pkg/cmd/server/apis/config"
+)
+
+func ConfigToFlags(kubeAPIServerConfig *configapi.MasterConfig) ([]string, error) {
+	args := map[string][]string{}
+	for key, slice := range kubeAPIServerConfig.KubernetesMasterConfig.APIServerArguments {
+		for _, val := range slice {
+			args[key] = append(args[key], val)
+		}
+	}
+
+	host, portString, err := net.SplitHostPort(kubeAPIServerConfig.ServingInfo.BindAddress)
+	if err != nil {
+		return nil, err
+	}
+
+	// these flags are overridden by a patch
+	// admission-control
+	// admission-control-config-file
+	// authentication-token-webhook-cache-ttl
+	// authentication-token-webhook-config-file
+	// authorization-mode
+	// authorization-policy-file
+	// authorization-webhook-cache-authorized-ttl
+	// authorization-webhook-cache-unauthorized-ttl
+	// authorization-webhook-config-file
+	// basic-auth-file
+	// disable-admission-plugins
+	// enable-admission-plugins
+	// enable-aggregator-routing
+	// enable-bootstrap-token-auth
+	// oidc-client-id
+	// oidc-groups-claim
+	// oidc-groups-prefix
+	// oidc-issuer-url
+	// oidc-required-claim
+	// oidc-signing-algs
+	// oidc-username-claim
+	// oidc-username-prefix
+	// service-account-lookup
+	// token-auth-file
+
+	// alsologtostderr - don't know whether to change it
+	// apiserver-count - ignored, hopefully we don't have to fix via patch
+	// cert-dir - ignored because we set certs
+
+	// these flags were never supported via config
+	// cloud-config
+	// cloud-provider
+	// cloud-provider-gce-lb-src-cidrs
+	// contention-profiling
+	// default-not-ready-toleration-seconds
+	// default-unreachable-toleration-seconds
+	// default-watch-cache-size
+	// delete-collection-workers
+	// deserialization-cache-size
+	// enable-garbage-collector
+	// etcd-compaction-interval
+	// etcd-count-metric-poll-period
+	// etcd-servers-overrides
+	// experimental-encryption-provider-config
+	// feature-gates
+	// http2-max-streams-per-connection
+	// insecure-bind-address
+	// kubelet-timeout
+	// log-backtrace-at
+	// log-dir
+	// log-flush-frequency
+	// logtostderr
+	// master-service-namespace
+	// max-connection-bytes-per-sec
+	// profiling
+	// request-timeout
+	// runtime-config
+	// service-account-api-audiences
+	// service-account-issuer
+	// service-account-key-file
+	// service-account-max-token-expiration
+	// service-account-signing-key-file
+	// stderrthreshold
+	// storage-versions
+	// target-ram-mb
+	// v
+	// version
+	// vmodule
+	// watch-cache
+	// watch-cache-sizes
+
+	setIfUnset(args, "allow-privileged", "true")
+	setIfUnset(args, "anonymous-auth", "false")
+	for flag, value := range auditFlags(kubeAPIServerConfig) {
+		setIfUnset(args, flag, value...)
+	}
+	setIfUnset(args, "bind-address", host)
+	setIfUnset(args, "client-ca-file", kubeAPIServerConfig.ServingInfo.ClientCA)
+	setIfUnset(args, "cors-allowed-origins", kubeAPIServerConfig.CORSAllowedOrigins...)
+	setIfUnset(args, "enable-logs-handler", "false")
+	setIfUnset(args, "enable-swagger-ui", "true")
+	setIfUnset(args, "endpoint-reconciler-type", "lease")
+	setIfUnset(args, "etcd-cafile", kubeAPIServerConfig.EtcdClientInfo.CA)
+	setIfUnset(args, "etcd-certfile", kubeAPIServerConfig.EtcdClientInfo.ClientCert.CertFile)
+	setIfUnset(args, "etcd-keyfile", kubeAPIServerConfig.EtcdClientInfo.ClientCert.KeyFile)
+	setIfUnset(args, "etcd-prefix", kubeAPIServerConfig.EtcdStorageConfig.KubernetesStoragePrefix)
+	setIfUnset(args, "etcd-servers", kubeAPIServerConfig.EtcdClientInfo.URLs...)
+	setIfUnset(args, "insecure-port", "0")
+	setIfUnset(args, "kubelet-certificate-authority", kubeAPIServerConfig.KubeletClientInfo.CA)
+	setIfUnset(args, "kubelet-client-certificate", kubeAPIServerConfig.KubeletClientInfo.ClientCert.CertFile)
+	setIfUnset(args, "kubelet-client-key", kubeAPIServerConfig.KubeletClientInfo.ClientCert.KeyFile)
+	setIfUnset(args, "kubelet-https", "true")
+	setIfUnset(args, "kubelet-preferred-address-types", "Hostname", "InternalIP", "ExternalIP")
+	setIfUnset(args, "kubelet-read-only-port", "0")
+	setIfUnset(args, "kubernetes-service-node-port", "0")
+	setIfUnset(args, "max-mutating-requests-inflight", fmt.Sprintf("%d", kubeAPIServerConfig.ServingInfo.MaxRequestsInFlight/2))
+	setIfUnset(args, "max-requests-inflight", fmt.Sprintf("%d", kubeAPIServerConfig.ServingInfo.MaxRequestsInFlight))
+	setIfUnset(args, "min-request-timeout", fmt.Sprintf("%d", kubeAPIServerConfig.ServingInfo.RequestTimeoutSeconds))
+	setIfUnset(args, "proxy-client-cert-file", kubeAPIServerConfig.AggregatorConfig.ProxyClientInfo.CertFile)
+	setIfUnset(args, "proxy-client-key-file", kubeAPIServerConfig.AggregatorConfig.ProxyClientInfo.KeyFile)
+	setIfUnset(args, "requestheader-allowed-names", kubeAPIServerConfig.AuthConfig.RequestHeader.ClientCommonNames...)
+	setIfUnset(args, "requestheader-client-ca-file", kubeAPIServerConfig.AuthConfig.RequestHeader.ClientCA)
+	setIfUnset(args, "requestheader-extra-headers-prefix", kubeAPIServerConfig.AuthConfig.RequestHeader.ExtraHeaderPrefixes...)
+	setIfUnset(args, "requestheader-group-headers", kubeAPIServerConfig.AuthConfig.RequestHeader.GroupHeaders...)
+	setIfUnset(args, "requestheader-username-headers", kubeAPIServerConfig.AuthConfig.RequestHeader.UsernameHeaders...)
+	setIfUnset(args, "secure-port", portString)
+	setIfUnset(args, "service-cluster-ip-range", kubeAPIServerConfig.KubernetesMasterConfig.ServicesSubnet)
+	setIfUnset(args, "service-node-port-range", kubeAPIServerConfig.KubernetesMasterConfig.ServicesNodePortRange)
+	setIfUnset(args, "storage-backend", "etcd3")
+	setIfUnset(args, "storage-media-type", "application/vnd.kubernetes.protobuf")
+	setIfUnset(args, "tls-cert-file", kubeAPIServerConfig.ServingInfo.ServerCert.CertFile)
+	setIfUnset(args, "tls-cipher-suites", kubeAPIServerConfig.ServingInfo.CipherSuites...)
+	setIfUnset(args, "tls-min-version", kubeAPIServerConfig.ServingInfo.MinTLSVersion)
+	setIfUnset(args, "tls-private-key-file", kubeAPIServerConfig.ServingInfo.ServerCert.KeyFile)
+	// TODO re-enable SNI for cluster up
+	// tls-sni-cert-key
+	setIfUnset(args, "secure-port", portString)
+	setIfUnset(args, "secure-port", portString)
+	setIfUnset(args, "secure-port", portString)
+
+	var keys []string
+	for key := range args {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	var arguments []string
+	for _, key := range keys {
+		for _, token := range args[key] {
+			arguments = append(arguments, fmt.Sprintf("--%s=%v", key, token))
+		}
+	}
+	return arguments, nil
+}
+
+// currently for cluster up, audit is just broken.
+// TODO fix this
+func auditFlags(kubeAPIServerConfig *configapi.MasterConfig) map[string][]string {
+	args := map[string][]string{}
+	for key, slice := range kubeAPIServerConfig.KubernetesMasterConfig.APIServerArguments {
+		for _, val := range slice {
+			args[key] = append(args[key], val)
+		}
+	}
+
+	return args
+}
+
+func setIfUnset(cmdLineArgs map[string][]string, key string, value ...string) {
+	if _, ok := cmdLineArgs[key]; !ok {
+		cmdLineArgs[key] = value
+	}
+}

pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/patch.go

@@ -0,0 +1,67 @@
+package openshiftkubeapiserver
+
+import (
+	configapi "github.com/openshift/origin/pkg/cmd/server/apis/config"
+	"k8s.io/apiserver/pkg/admission"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	clientgoinformers "k8s.io/client-go/informers"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app"
+	internalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
+	"k8s.io/kubernetes/pkg/master"
+
+	oauthclient "github.com/openshift/client-go/oauth/clientset/versioned"
+	oauthinformer "github.com/openshift/client-go/oauth/informers/externalversions"
+	userclient "github.com/openshift/client-go/user/clientset/versioned"
+	userinformer "github.com/openshift/client-go/user/informers/externalversions"
+
+	"time"
+)
+
+type KubeAPIServerServerPatchContext struct {
+	initialized bool
+
+	postStartHooks     map[string]genericapiserver.PostStartHookFunc
+	informerStartFuncs []func(stopCh <-chan struct{})
+}
+
+func NewOpenShiftKubeAPIServerConfigPatch(kubeAPIServerConfig *configapi.MasterConfig) (app.KubeAPIServerConfigFunc, *KubeAPIServerServerPatchContext) {
+	patchContext := &KubeAPIServerServerPatchContext{}
+	return func(config *master.Config, internalInformers internalinformers.SharedInformerFactory, versionedInformers clientgoinformers.SharedInformerFactory, pluginInitializers *[]admission.PluginInitializer) error {
+		userClient, err := userclient.NewForConfig(config.GenericConfig.LoopbackClientConfig)
+		if err != nil {
+			return err
+		}
+		userInformer := userinformer.NewSharedInformerFactory(userClient, 10*time.Minute)
+		patchContext.informerStartFuncs = append(patchContext.informerStartFuncs, userInformer.Start)
+		oauthClient, err := oauthclient.NewForConfig(config.GenericConfig.LoopbackClientConfig)
+		if err != nil {
+			return err
+		}
+		oauthInformer := oauthinformer.NewSharedInformerFactory(oauthClient, 10*time.Minute)
+		patchContext.informerStartFuncs = append(patchContext.informerStartFuncs, oauthInformer.Start)
+
+		authenticator, postStartHooks, err := NewAuthenticator(*kubeAPIServerConfig, config.GenericConfig.LoopbackClientConfig, oauthInformer.Oauth().V1().OAuthClients().Lister(), userInformer.User().V1().Groups())
+		if err != nil {
+			return err
+		}
+		config.GenericConfig.Authentication.Authenticator = authenticator
+		for key, fn := range postStartHooks {
+			patchContext.postStartHooks[key] = fn
+		}
+
+		authorizer := NewAuthorizer(internalInformers, versionedInformers)
+		config.GenericConfig.Authorization.Authorizer = authorizer
+
+		patchContext.initialized = true
+
+		return nil
+	}, patchContext
+}
+
+func (c *KubeAPIServerServerPatchContext) PatchServer(server *master.Master) error {
+	for name, fn := range c.postStartHooks {
+		server.GenericAPIServer.AddPostStartHookOrDie(name, fn)
+	}
+
+	return nil
+}