add kube-apiserver wiring

pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/flags.go

@@ -0,0 +1,179 @@
+package openshiftkubeapiserver
+
+import (
+	"fmt"
+	"net"
+	"sort"
+
+	configapi "github.com/openshift/origin/pkg/cmd/server/apis/config"
+)
+
+func ConfigToFlags(kubeAPIServerConfig *configapi.MasterConfig) ([]string, error) {
+	args := map[string][]string{}
+	for key, slice := range kubeAPIServerConfig.KubernetesMasterConfig.APIServerArguments {
+		for _, val := range slice {
+			args[key] = append(args[key], val)
+		}
+	}
+
+	host, portString, err := net.SplitHostPort(kubeAPIServerConfig.ServingInfo.BindAddress)
+	if err != nil {
+		return nil, err
+	}
+
+	// these flags are overridden by a patch
+	// admission-control
+	// authentication-token-webhook-cache-ttl
+	// authentication-token-webhook-config-file
+	// authorization-mode
+	// authorization-policy-file
+	// authorization-webhook-cache-authorized-ttl
+	// authorization-webhook-cache-unauthorized-ttl
+	// authorization-webhook-config-file
+	// basic-auth-file
+	// enable-aggregator-routing
+	// enable-bootstrap-token-auth
+	// oidc-client-id
+	// oidc-groups-claim
+	// oidc-groups-prefix
+	// oidc-issuer-url
+	// oidc-required-claim
+	// oidc-signing-algs
+	// oidc-username-claim
+	// oidc-username-prefix
+	// service-account-lookup
+	// token-auth-file
+
+	// alsologtostderr - don't know whether to change it
+	// apiserver-count - ignored, hopefully we don't have to fix via patch
+	// cert-dir - ignored because we set certs
+
+	// these flags were never supported via config
+	// cloud-config
+	// cloud-provider
+	// cloud-provider-gce-lb-src-cidrs
+	// contention-profiling
+	// default-not-ready-toleration-seconds
+	// default-unreachable-toleration-seconds
+	// default-watch-cache-size
+	// delete-collection-workers
+	// deserialization-cache-size
+	// enable-garbage-collector
+	// etcd-compaction-interval
+	// etcd-count-metric-poll-period
+	// etcd-servers-overrides
+	// experimental-encryption-provider-config
+	// feature-gates
+	// http2-max-streams-per-connection
+	// insecure-bind-address
+	// kubelet-timeout
+	// log-backtrace-at
+	// log-dir
+	// log-flush-frequency
+	// logtostderr
+	// master-service-namespace
+	// max-connection-bytes-per-sec
+	// profiling
+	// request-timeout
+	// runtime-config
+	// service-account-api-audiences
+	// service-account-issuer
+	// service-account-key-file
+	// service-account-max-token-expiration
+	// service-account-signing-key-file
+	// stderrthreshold
+	// storage-versions
+	// target-ram-mb
+	// v
+	// version
+	// vmodule
+	// watch-cache
+	// watch-cache-sizes
+
+	// TODO, we need to set these in order to enable the right admission plugins in each of the servers
+	// TODO this is needed for a viable cluster up
+	//setIfUnset(args, "admission-control-config-file", "AdmissionConfiguration file")
+	//setIfUnset(args, "disable-admission-plugins", "AdmissionConfiguration file")
+	//setIfUnset(args, "enable-admission-plugins", "AdmissionConfiguration file")
+
+	setIfUnset(args, "allow-privileged", "true")
+	setIfUnset(args, "anonymous-auth", "false")
+	setIfUnset(args, "authorization-mode", "RBAC") // overridden later, but this runs the poststarthook for bootstrapping RBAC
+	for flag, value := range auditFlags(kubeAPIServerConfig) {
+		setIfUnset(args, flag, value...)
+	}
+	setIfUnset(args, "bind-address", host)
+	setIfUnset(args, "client-ca-file", kubeAPIServerConfig.ServingInfo.ClientCA)
+	setIfUnset(args, "cors-allowed-origins", kubeAPIServerConfig.CORSAllowedOrigins...)
+	setIfUnset(args, "enable-logs-handler", "false")
+	setIfUnset(args, "enable-swagger-ui", "true")
+	setIfUnset(args, "endpoint-reconciler-type", "lease")
+	setIfUnset(args, "etcd-cafile", kubeAPIServerConfig.EtcdClientInfo.CA)
+	setIfUnset(args, "etcd-certfile", kubeAPIServerConfig.EtcdClientInfo.ClientCert.CertFile)
+	setIfUnset(args, "etcd-keyfile", kubeAPIServerConfig.EtcdClientInfo.ClientCert.KeyFile)
+	setIfUnset(args, "etcd-prefix", kubeAPIServerConfig.EtcdStorageConfig.KubernetesStoragePrefix)
+	setIfUnset(args, "etcd-servers", kubeAPIServerConfig.EtcdClientInfo.URLs...)
+	setIfUnset(args, "insecure-port", "0")
+	setIfUnset(args, "kubelet-certificate-authority", kubeAPIServerConfig.KubeletClientInfo.CA)
+	setIfUnset(args, "kubelet-client-certificate", kubeAPIServerConfig.KubeletClientInfo.ClientCert.CertFile)
+	setIfUnset(args, "kubelet-client-key", kubeAPIServerConfig.KubeletClientInfo.ClientCert.KeyFile)
+	setIfUnset(args, "kubelet-https", "true")
+	setIfUnset(args, "kubelet-preferred-address-types", "Hostname", "InternalIP", "ExternalIP")
+	setIfUnset(args, "kubelet-read-only-port", "0")
+	setIfUnset(args, "kubernetes-service-node-port", "0")
+	setIfUnset(args, "max-mutating-requests-inflight", fmt.Sprintf("%d", kubeAPIServerConfig.ServingInfo.MaxRequestsInFlight/2))
+	setIfUnset(args, "max-requests-inflight", fmt.Sprintf("%d", kubeAPIServerConfig.ServingInfo.MaxRequestsInFlight))
+	setIfUnset(args, "min-request-timeout", fmt.Sprintf("%d", kubeAPIServerConfig.ServingInfo.RequestTimeoutSeconds))
+	setIfUnset(args, "proxy-client-cert-file", kubeAPIServerConfig.AggregatorConfig.ProxyClientInfo.CertFile)
+	setIfUnset(args, "proxy-client-key-file", kubeAPIServerConfig.AggregatorConfig.ProxyClientInfo.KeyFile)
+	setIfUnset(args, "requestheader-allowed-names", kubeAPIServerConfig.AuthConfig.RequestHeader.ClientCommonNames...)
+	setIfUnset(args, "requestheader-client-ca-file", kubeAPIServerConfig.AuthConfig.RequestHeader.ClientCA)
+	setIfUnset(args, "requestheader-extra-headers-prefix", kubeAPIServerConfig.AuthConfig.RequestHeader.ExtraHeaderPrefixes...)
+	setIfUnset(args, "requestheader-group-headers", kubeAPIServerConfig.AuthConfig.RequestHeader.GroupHeaders...)
+	setIfUnset(args, "requestheader-username-headers", kubeAPIServerConfig.AuthConfig.RequestHeader.UsernameHeaders...)
+	setIfUnset(args, "secure-port", portString)
+	setIfUnset(args, "service-cluster-ip-range", kubeAPIServerConfig.KubernetesMasterConfig.ServicesSubnet)
+	setIfUnset(args, "service-node-port-range", kubeAPIServerConfig.KubernetesMasterConfig.ServicesNodePortRange)
+	setIfUnset(args, "storage-backend", "etcd3")
+	setIfUnset(args, "storage-media-type", "application/vnd.kubernetes.protobuf")
+	setIfUnset(args, "tls-cert-file", kubeAPIServerConfig.ServingInfo.ServerCert.CertFile)
+	setIfUnset(args, "tls-cipher-suites", kubeAPIServerConfig.ServingInfo.CipherSuites...)
+	setIfUnset(args, "tls-min-version", kubeAPIServerConfig.ServingInfo.MinTLSVersion)
+	setIfUnset(args, "tls-private-key-file", kubeAPIServerConfig.ServingInfo.ServerCert.KeyFile)
+	// TODO re-enable SNI for cluster up
+	// tls-sni-cert-key
+	setIfUnset(args, "secure-port", portString)
+
+	var keys []string
+	for key := range args {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	var arguments []string
+	for _, key := range keys {
+		for _, token := range args[key] {
+			arguments = append(arguments, fmt.Sprintf("--%s=%v", key, token))
+		}
+	}
+	return arguments, nil
+}
+
+// currently for cluster up, audit is just broken.
+// TODO fix this
+func auditFlags(kubeAPIServerConfig *configapi.MasterConfig) map[string][]string {
+	args := map[string][]string{}
+	for key, slice := range kubeAPIServerConfig.KubernetesMasterConfig.APIServerArguments {
+		for _, val := range slice {
+			args[key] = append(args[key], val)
+		}
+	}
+
+	return args
+}
+
+func setIfUnset(cmdLineArgs map[string][]string, key string, value ...string) {
+	if _, ok := cmdLineArgs[key]; !ok {
+		cmdLineArgs[key] = value
+	}
+}

pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/patch.go

@@ -10,6 +10,7 @@ import (
 	kexternalinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/restmapper"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app"
 	internalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
 	kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
 	"k8s.io/kubernetes/pkg/master"
@@ -43,9 +44,7 @@ type KubeAPIServerServerPatchContext struct {
 	informerStartFuncs []func(stopCh <-chan struct{})
 }
 
-type KubeAPIServerConfigFunc func(config *master.Config, internalInformers internalinformers.SharedInformerFactory, kubeInformers clientgoinformers.SharedInformerFactory, pluginInitializers *[]admission.PluginInitializer, stopCh <-chan struct{}) (genericapiserver.DelegationTarget, error)
-
-func NewOpenShiftKubeAPIServerConfigPatch(delegateAPIServer genericapiserver.DelegationTarget, kubeAPIServerConfig *configapi.MasterConfig) (KubeAPIServerConfigFunc, *KubeAPIServerServerPatchContext) {
+func NewOpenShiftKubeAPIServerConfigPatch(delegateAPIServer genericapiserver.DelegationTarget, kubeAPIServerConfig *configapi.MasterConfig) (app.KubeAPIServerConfigFunc, *KubeAPIServerServerPatchContext) {
 	patchContext := &KubeAPIServerServerPatchContext{
 		postStartHooks: map[string]genericapiserver.PostStartHookFunc{},
 	}

pkg/cmd/openshift-kube-apiserver/server.go

@@ -1,20 +1,20 @@
 package openshift_kube_apiserver
 
 import (
+	"fmt"
+
 	"github.com/golang/glog"
 
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	utilwait "k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/pkg/version"
-	aggregatorinstall "k8s.io/kube-aggregator/pkg/apis/apiregistration/install"
-	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app"
 	"k8s.io/kubernetes/pkg/capabilities"
 	kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
 
+	"github.com/openshift/origin/pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver"
 	configapi "github.com/openshift/origin/pkg/cmd/server/apis/config"
 	"github.com/openshift/origin/pkg/cmd/server/apis/config/validation"
-	"github.com/openshift/origin/pkg/cmd/server/origin"
-	"github.com/openshift/origin/pkg/cmd/util/variable"
 )
 
 func RunOpenShiftKubeAPIServerServer(masterConfig *configapi.MasterConfig) error {
@@ -28,11 +28,6 @@ func RunOpenShiftKubeAPIServerServer(masterConfig *configapi.MasterConfig) error
 		},
 	})
 
-	// install aggregator types into the scheme so that "normal" RESTOptionsGetters can work for us.
-	// done in Start() prior to doing any other initialization so we don't mutate the scheme after it is being used by clients in other goroutines.
-	// TODO: make scheme threadsafe and do this as part of aggregator config building
-	aggregatorinstall.Install(legacyscheme.Scheme)
-
 	validationResults := validation.ValidateMasterConfig(masterConfig, nil)
 	if len(validationResults.Warnings) != 0 {
 		for _, warning := range validationResults.Warnings {
@@ -43,22 +38,22 @@ func RunOpenShiftKubeAPIServerServer(masterConfig *configapi.MasterConfig) error
 		return kerrors.NewInvalid(configapi.Kind("MasterConfig"), "master-config.yaml", validationResults.Errors)
 	}
 
-	informers := origin.InformerAccess(nil) // use real kube-apiserver loopback client with secret token instead of that from masterConfig.MasterClients.OpenShiftLoopbackKubeConfig
-	openshiftConfig, err := origin.BuildMasterConfig(*masterConfig, informers)
+	configPatchFn, serverPatchContext := openshiftkubeapiserver.NewOpenShiftKubeAPIServerConfigPatch(genericapiserver.NewEmptyDelegate(), masterConfig)
+	app.OpenShiftKubeAPIServerConfigPatch = configPatchFn
+	app.OpenShiftKubeAPIServerServerPatch = serverPatchContext.PatchServer
+
+	cmd := app.NewAPIServerCommand(utilwait.NeverStop)
+	args, err := openshiftkubeapiserver.ConfigToFlags(masterConfig)
 	if err != nil {
 		return err
 	}
-
-	glog.Infof("Starting master on %s (%s)", masterConfig.ServingInfo.BindAddress, version.Get().String())
-	glog.Infof("Public master address is %s", masterConfig.MasterPublicURL)
-	imageTemplate := variable.NewDefaultImageTemplate()
-	imageTemplate.Format = masterConfig.ImageConfig.Format
-	imageTemplate.Latest = masterConfig.ImageConfig.Latest
-	glog.Infof("Using images from %q", imageTemplate.ExpandOrDie("<component>"))
-
-	if err := openshiftConfig.RunKubeAPIServer(utilwait.NeverStop); err != nil {
+	if err := cmd.ParseFlags(args); err != nil {
+		return err
+	}
+	glog.Infof("`kube-apiserver %v`", args)
+	if err := cmd.RunE(cmd, nil); err != nil {
 		return err
 	}
 
-	return nil
+	return fmt.Errorf("`kube-apiserver %v` exited", args)
 }