add nodeselector and annotation build pod overrides and defaulters

api/swagger-spec/oapi-v1.json

@@ -21462,6 +21462,10 @@
       "type": "integer",
       "format": "int64",
       "description": "completionDeadlineSeconds is an optional duration in seconds, counted from the time when a build pod gets scheduled in the system, that the build may be active on a node before the system actively tries to terminate the build; value must be positive integer"
+     },
+     "nodeSelector": {
+      "type": "object",
+      "description": "nodeSelector is a selector which must be true for the build pod to fit on a node"
      }
     }
    },
@@ -22513,6 +22517,10 @@
       "format": "int64",
       "description": "completionDeadlineSeconds is an optional duration in seconds, counted from the time when a build pod gets scheduled in the system, that the build may be active on a node before the system actively tries to terminate the build; value must be positive integer"
      },
+     "nodeSelector": {
+      "type": "object",
+      "description": "nodeSelector is a selector which must be true for the build pod to fit on a node"
+     },
      "triggeredBy": {
       "type": "array",
       "items": {

api/swagger-spec/openshift-openapi-spec.json

@@ -44680,6 +44680,10 @@
       "type": "integer",
       "format": "int64"
      },
+     "nodeSelector": {
+      "description": "nodeSelector is a selector which must be true for the build pod to fit on a node",
+      "type": "object"
+     },
      "output": {
       "$ref": "#/definitions/v1.BuildOutput"
      },
@@ -44913,6 +44917,10 @@
       "type": "integer",
       "format": "int64"
      },
+     "nodeSelector": {
+      "description": "nodeSelector is a selector which must be true for the build pod to fit on a node",
+      "type": "object"
+     },
      "output": {
       "$ref": "#/definitions/v1.BuildOutput"
      },

docs/man/man1/openshift-start-kubernetes-apiserver.1

@@ -26,7 +26,7 @@ This command launches an instance of the Kubernetes apiserver (kube\-apiserver).
 
 .PP
 \fB\-\-admission\-control\fP="AlwaysAdmit"
-    Ordered list of plug\-ins to do admission control of resources into cluster. Comma\-delimited list of: AlwaysAdmit, AlwaysDeny, AlwaysPullImages, BuildByStrategy, BuildDefaults, BuildOverrides, ClusterResourceOverride, DefaultStorageClass, DenyEscalatingExec, DenyExecOnPrivileged, ExternalIPRanger, ImagePolicyWebhook, InitialResources, LimitPodHardAntiAffinityTopology, LimitRanger, NamespaceAutoProvision, NamespaceExists, NamespaceLifecycle, OriginNamespaceLifecycle, OriginPodNodeEnvironment, PersistentVolumeLabel, PodNodeConstraints, PodSecurityPolicy, ProjectRequestLimit, ResourceQuota, RunOnceDuration, SCCExecRestrictions, SecurityContextConstraint, SecurityContextDeny, ServiceAccount, openshift.io/ClusterResourceQuota, openshift.io/ImageLimitRange, openshift.io/ImagePolicy, openshift.io/JenkinsBootstrapper, openshift.io/OriginResourceQuota, openshift.io/OwnerReference, openshift.io/RestrictedEndpointsAdmission.
+    Ordered list of plug\-ins to do admission control of resources into cluster. Comma\-delimited list of: AlwaysAdmit, AlwaysDeny, AlwaysPullImages, BuildByStrategy, ClusterResourceOverride, DefaultStorageClass, DenyEscalatingExec, DenyExecOnPrivileged, ExternalIPRanger, ImagePolicyWebhook, InitialResources, LimitPodHardAntiAffinityTopology, LimitRanger, NamespaceAutoProvision, NamespaceExists, NamespaceLifecycle, OriginNamespaceLifecycle, OriginPodNodeEnvironment, PersistentVolumeLabel, PodNodeConstraints, PodSecurityPolicy, ProjectRequestLimit, ResourceQuota, RunOnceDuration, SCCExecRestrictions, SecurityContextConstraint, SecurityContextDeny, ServiceAccount, openshift.io/ClusterResourceQuota, openshift.io/ImageLimitRange, openshift.io/ImagePolicy, openshift.io/JenkinsBootstrapper, openshift.io/OriginResourceQuota, openshift.io/OwnerReference, openshift.io/RestrictedEndpointsAdmission.
 
 .PP
 \fB\-\-admission\-control\-config\-file\fP=""

pkg/build/admission/buildpodutil.go

@@ -4,64 +4,29 @@ import (
 	"errors"
 	"fmt"
 
-	"k8s.io/kubernetes/pkg/admission"
 	kapi "k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/unversioned"
 	"k8s.io/kubernetes/pkg/runtime"
 
 	buildapi "github.com/openshift/origin/pkg/build/api"
 )
 
-// IsBuildPod returns true if a pod is a pod generated for a Build
-func IsBuildPod(a admission.Attributes) bool {
-	if a.GetResource().GroupResource() != kapi.Resource("pods") {
-		return false
-	}
-	if len(a.GetSubresource()) != 0 {
-		return false
-	}
-	pod, err := GetPod(a)
-	if err != nil {
-		return false
-	}
-	return hasBuildAnnotation(pod) && hasBuildEnvVar(pod)
-}
-
 // GetBuild returns a build object encoded in a pod's BUILD environment variable along with
 // its encoding version
-func GetBuild(a admission.Attributes) (*buildapi.Build, unversioned.GroupVersion, error) {
-	pod, err := GetPod(a)
-	if err != nil {
-		return nil, unversioned.GroupVersion{}, err
-	}
+func GetBuildFromPod(pod *kapi.Pod) (*buildapi.Build, unversioned.GroupVersion, error) {
 	build, version, err := getBuildFromPod(pod)
 	if err != nil {
-		return nil, unversioned.GroupVersion{}, admission.NewForbidden(a, fmt.Errorf("unable to get build from pod: %v", err))
+		return nil, unversioned.GroupVersion{}, fmt.Errorf("unable to get build from pod: %v", err)
 	}
 	return build, version, nil
 }
 
-// GetPod returns a pod from an admission attributes object
-func GetPod(a admission.Attributes) (*kapi.Pod, error) {
-	pod, isPod := a.GetObject().(*kapi.Pod)
-	if !isPod {
-		return nil, admission.NewForbidden(a, fmt.Errorf("unrecognized request object: %#v", a.GetObject()))
-	}
-	return pod, nil
-}
-
 // SetBuild encodes a build object and sets it in a pod's BUILD environment variable
-func SetBuild(a admission.Attributes, build *buildapi.Build, groupVersion unversioned.GroupVersion) error {
-	pod, err := GetPod(a)
-	if err != nil {
-		return err
-	}
-
-	err = setBuildInPod(build, pod, groupVersion)
+func SetBuildInPod(pod *kapi.Pod, build *buildapi.Build, groupVersion unversioned.GroupVersion) error {
+	err := setBuildInPod(build, pod, groupVersion)
 	if err != nil {
-		return admission.NewForbidden(a, fmt.Errorf("unable to set build in pod: %v", err))
+		return fmt.Errorf("unable to set build in pod: %v", err)
 	}
-
 	return nil
 }
 
@@ -70,12 +35,7 @@ func SetBuild(a admission.Attributes, build *buildapi.Build, groupVersion unvers
 // environment variable may have been set in multiple ways: a default value,
 // by a BuildConfig, or by the BuildDefaults admission plugin. In this method
 // we finally act on the value by injecting it into the Pod.
-func SetBuildLogLevel(attributes admission.Attributes, build *buildapi.Build) error {
-	pod, err := GetPod(attributes)
-	if err != nil {
-		return err
-	}
-
+func SetPodLogLevelFromBuild(pod *kapi.Pod, build *buildapi.Build) error {
 	var envs []kapi.EnvVar
 
 	// Check whether the build strategy supports --loglevel parameter.

pkg/build/admission/config.go

@@ -1,25 +1,35 @@
 package admission
 
 import (
-	"io"
 	"io/ioutil"
-	"reflect"
+	"os"
+
+	"github.com/golang/glog"
 
 	"k8s.io/kubernetes/pkg/runtime"
 
+	configapi "github.com/openshift/origin/pkg/cmd/server/api"
 	configlatest "github.com/openshift/origin/pkg/cmd/server/api/latest"
+	"github.com/openshift/origin/pkg/cmd/util/pluginconfig"
 )
 
 // ReadPluginConfig will read a plugin configuration object from a reader stream
-func ReadPluginConfig(reader io.Reader, config runtime.Object) error {
-	if reader == nil || reflect.ValueOf(reader).IsNil() {
+func ReadPluginConfig(pluginConfig map[string]configapi.AdmissionPluginConfig, name string, config runtime.Object) error {
+
+	configFilePath, err := pluginconfig.GetPluginConfigFile(pluginConfig, name, "")
+	if configFilePath == "" {
 		return nil
 	}
-
-	configBytes, err := ioutil.ReadAll(reader)
+	configData, err := os.Open(configFilePath)
 	if err != nil {
+		glog.Fatalf("Couldn't open plugin configuration %s: %#v", configFilePath, err)
 		return err
 	}
+
+	defer configData.Close()
+
+	configBytes, err := ioutil.ReadAll(configData)
+
 	err = configlatest.ReadYAMLInto(configBytes, config)
 	if err != nil {
 		return err

pkg/build/admission/defaults/admission.go

@@ -1,100 +1,91 @@
 package defaults
 
 import (
-	"io"
-
 	"github.com/golang/glog"
-	"k8s.io/kubernetes/pkg/admission"
 	kapi "k8s.io/kubernetes/pkg/api"
-	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 
 	buildadmission "github.com/openshift/origin/pkg/build/admission"
 	defaultsapi "github.com/openshift/origin/pkg/build/admission/defaults/api"
 	"github.com/openshift/origin/pkg/build/admission/defaults/api/validation"
 	buildapi "github.com/openshift/origin/pkg/build/api"
+	configapi "github.com/openshift/origin/pkg/cmd/server/api"
 )
 
-func init() {
-	admission.RegisterPlugin("BuildDefaults", func(c clientset.Interface, config io.Reader) (admission.Interface, error) {
-
-		defaultsConfig, err := getConfig(config)
-		if err != nil {
-			return nil, err
-		}
-
-		glog.V(5).Infof("Initializing BuildDefaults plugin with config: %#v", defaultsConfig)
-		return NewBuildDefaults(defaultsConfig), nil
-	})
+type BuildDefaults struct {
+	config *defaultsapi.BuildDefaultsConfig
 }
 
-func getConfig(in io.Reader) (*defaultsapi.BuildDefaultsConfig, error) {
-	defaultsConfig := &defaultsapi.BuildDefaultsConfig{}
-	err := buildadmission.ReadPluginConfig(in, defaultsConfig)
+// NewBuildDefaults creates a new BuildDefaults that will apply the defaults specified in the plugin config
+func NewBuildDefaults(pluginConfig map[string]configapi.AdmissionPluginConfig) (BuildDefaults, error) {
+	config := &defaultsapi.BuildDefaultsConfig{}
+	err := buildadmission.ReadPluginConfig(pluginConfig, defaultsapi.BuildDefaultsPlugin, config)
 	if err != nil {
-		return nil, err
+		return BuildDefaults{}, err
 	}
-	errs := validation.ValidateBuildDefaultsConfig(defaultsConfig)
+	errs := validation.ValidateBuildDefaultsConfig(config)
 	if len(errs) > 0 {
-		return nil, errs.ToAggregate()
+		return BuildDefaults{}, errs.ToAggregate()
 	}
-	return defaultsConfig, nil
+	glog.V(4).Infof("Initialized build defaults plugin with config: %#v", *config)
+	return BuildDefaults{config: config}, nil
 }
 
-type buildDefaults struct {
-	*admission.Handler
-	defaultsConfig *defaultsapi.BuildDefaultsConfig
-}
-
-// NewBuildDefaults returns an admission control for builds that sets build defaults
-// based on the plugin configuration
-func NewBuildDefaults(defaultsConfig *defaultsapi.BuildDefaultsConfig) admission.Interface {
-	return &buildDefaults{
-		Handler:        admission.NewHandler(admission.Create),
-		defaultsConfig: defaultsConfig,
-	}
-}
-
-// Admit applies configured build defaults to a pod that is identified
-// as a build pod.
-func (a *buildDefaults) Admit(attributes admission.Attributes) error {
-	if a.defaultsConfig == nil {
-		return nil
-	}
-	if !buildadmission.IsBuildPod(attributes) {
+// ApplyDefaults applies configured build defaults to a build pod
+func (b BuildDefaults) ApplyDefaults(pod *kapi.Pod) error {
+	if b.config == nil {
 		return nil
 	}
-	build, version, err := buildadmission.GetBuild(attributes)
+
+	build, version, err := buildadmission.GetBuildFromPod(pod)
 	if err != nil {
 		return nil
 	}
 
-	glog.V(4).Infof("Handling build %s/%s", build.Namespace, build.Name)
+	glog.V(4).Infof("Applying defaults to build %s/%s", build.Namespace, build.Name)
+
+	b.applyBuildDefaults(build)
 
-	a.applyBuildDefaults(build)
+	b.applyPodDefaults(pod)
 
-	err = buildadmission.SetBuildLogLevel(attributes, build)
+	err = buildadmission.SetPodLogLevelFromBuild(pod, build)
 	if err != nil {
 		return err
 	}
 
-	return buildadmission.SetBuild(attributes, build, version)
+	return buildadmission.SetBuildInPod(pod, build, version)
 }
 
-func (a *buildDefaults) applyBuildDefaults(build *buildapi.Build) {
+func (b BuildDefaults) applyPodDefaults(pod *kapi.Pod) {
+	if len(b.config.NodeSelector) != 0 && pod.Spec.NodeSelector == nil {
+		pod.Spec.NodeSelector = map[string]string{}
+	}
+	for k, v := range b.config.NodeSelector {
+		addDefaultNodeSelector(k, v, pod.Spec.NodeSelector)
+	}
+
+	if len(b.config.Annotations) != 0 && pod.Annotations == nil {
+		pod.Annotations = map[string]string{}
+	}
+	for k, v := range b.config.Annotations {
+		addDefaultAnnotations(k, v, pod.Annotations)
+	}
+}
+
+func (b BuildDefaults) applyBuildDefaults(build *buildapi.Build) {
 	// Apply default env
 	buildEnv := getBuildEnv(build)
-	for _, envVar := range a.defaultsConfig.Env {
+	for _, envVar := range b.config.Env {
 		glog.V(5).Infof("Adding default environment variable %s=%s to build %s/%s", envVar.Name, envVar.Value, build.Namespace, build.Name)
 		addDefaultEnvVar(envVar, buildEnv)
 	}
 
 	// Apply default labels
-	for _, lbl := range a.defaultsConfig.ImageLabels {
+	for _, lbl := range b.config.ImageLabels {
 		glog.V(5).Infof("Adding default image label %s=%s to build %s/%s", lbl.Name, lbl.Value, build.Namespace, build.Name)
 		addDefaultLabel(lbl, &build.Spec.Output.ImageLabels)
 	}
 
-	sourceDefaults := a.defaultsConfig.SourceStrategyDefaults
+	sourceDefaults := b.config.SourceStrategyDefaults
 	sourceStrategy := build.Spec.Strategy.SourceStrategy
 	if sourceDefaults != nil && sourceDefaults.Incremental != nil && *sourceDefaults.Incremental &&
 		sourceStrategy != nil && sourceStrategy.Incremental == nil {
@@ -107,25 +98,25 @@ func (a *buildDefaults) applyBuildDefaults(build *buildapi.Build) {
 	if build.Spec.Source.Git == nil {
 		return
 	}
-	if len(a.defaultsConfig.GitHTTPProxy) != 0 {
+	if len(b.config.GitHTTPProxy) != 0 {
 		if build.Spec.Source.Git.HTTPProxy == nil {
-			t := a.defaultsConfig.GitHTTPProxy
+			t := b.config.GitHTTPProxy
 			glog.V(5).Infof("Setting default Git HTTP proxy of build %s/%s to %s", build.Namespace, build.Name, t)
 			build.Spec.Source.Git.HTTPProxy = &t
 		}
 	}
 
-	if len(a.defaultsConfig.GitHTTPSProxy) != 0 {
+	if len(b.config.GitHTTPSProxy) != 0 {
 		if build.Spec.Source.Git.HTTPSProxy == nil {
-			t := a.defaultsConfig.GitHTTPSProxy
+			t := b.config.GitHTTPSProxy
 			glog.V(5).Infof("Setting default Git HTTPS proxy of build %s/%s to %s", build.Namespace, build.Name, t)
 			build.Spec.Source.Git.HTTPSProxy = &t
 		}
 	}
 
-	if len(a.defaultsConfig.GitNoProxy) != 0 {
+	if len(b.config.GitNoProxy) != 0 {
 		if build.Spec.Source.Git.NoProxy == nil {
-			t := a.defaultsConfig.GitNoProxy
+			t := b.config.GitNoProxy
 			glog.V(5).Infof("Setting default Git no proxy of build %s/%s to %s", build.Namespace, build.Name, t)
 			build.Spec.Source.Git.NoProxy = &t
 		}
@@ -171,3 +162,19 @@ func addDefaultLabel(defaultLabel buildapi.ImageLabel, buildLabels *[]buildapi.I
 		*buildLabels = append(*buildLabels, defaultLabel)
 	}
 }
+
+func addDefaultNodeSelector(k, v string, selectors map[string]string) bool {
+	if _, ok := selectors[k]; !ok {
+		selectors[k] = v
+		return true
+	}
+	return false
+}
+
+func addDefaultAnnotations(k, v string, annotations map[string]string) bool {
+	if _, ok := annotations[k]; !ok {
+		annotations[k] = v
+		return true
+	}
+	return false
+}