rebase 1.9.0 beta.1

pkg/cmd/server/kubernetes/node/node.go

@@ -9,17 +9,17 @@ import (
 	"path/filepath"
 	"time"
 
-	dockertypes "github.com/docker/docker/api/types"
 	dockerclient "github.com/fsouza/go-dockerclient"
 	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/kubelet/dockershim"
+	dockertools "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker"
 
 	kapiv1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeletapp "k8s.io/kubernetes/cmd/kubelet/app"
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
 	cadvisortesting "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
-	dockertools "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker"
 	"k8s.io/kubernetes/pkg/volume"
 
 	configapi "github.com/openshift/origin/pkg/cmd/server/api"
@@ -34,7 +34,7 @@ const minimumDockerAPIVersionWithPullByID = "1.22"
 // All errors here are fatal.
 func (c *NodeConfig) EnsureKubeletAccess() {
 	if _, err := os.Stat("/var/lib/docker"); os.IsPermission(err) {
-		c.HandleDockerError("Unable to view the /var/lib/docker directory - are you running as root?")
+		glog.Fatal("Unable to view the /var/lib/docker directory - are you running as root?")
 	}
 	if c.Containerized {
 		if _, err := os.Stat("/rootfs"); os.IsPermission(err) || os.IsNotExist(err) {
@@ -82,73 +82,67 @@ func (c *NodeConfig) EnsureDocker(docker *dockerutil.Helper) {
 	if c.KubeletServer.ContainerRuntime != "docker" {
 		return
 	}
-	dockerClient, dockerAddr, err := docker.GetKubeClient(c.KubeletServer.RuntimeRequestTimeout.Duration, c.KubeletServer.ImagePullProgressDeadline.Duration)
-	if err != nil {
-		c.HandleDockerError(fmt.Sprintf("Unable to create a Docker client for %s - Docker must be installed and running to start containers.\n%v", dockerAddr, err))
-		return
+
+	var endpoint string
+	if len(os.Getenv("DOCKER_HOST")) > 0 {
+		endpoint = os.Getenv("DOCKER_HOST")
+	} else {
+		endpoint = "unix:///var/run/docker.sock"
+	}
+
+	dockerClientConfig := &dockershim.ClientConfig{
+		DockerEndpoint:            endpoint,
+		RuntimeRequestTimeout:     c.KubeletServer.RuntimeRequestTimeout.Duration,
+		ImagePullProgressDeadline: c.KubeletServer.ImagePullProgressDeadline.Duration,
 	}
-	if url, err := url.Parse(dockerAddr); err == nil && url.Scheme == "unix" && len(url.Path) > 0 {
+	client := dockertools.ConnectToDockerOrDie(endpoint, c.KubeletServer.RuntimeRequestTimeout.Duration, c.KubeletServer.ImagePullProgressDeadline.Duration, false, false)
+	dockerClient := &dockerutil.KubeDocker{client}
+
+	if url, err := url.Parse(endpoint); err == nil && url.Scheme == "unix" && len(url.Path) > 0 {
 		s, err := os.Stat(url.Path)
 		switch {
 		case os.IsNotExist(err):
-			c.HandleDockerError(fmt.Sprintf("No Docker socket found at %s. Have you started the Docker daemon?", url.Path))
+			glog.Fatalf("No Docker socket found at %s. Have you started the Docker daemon?", url.Path)
 			return
 		case os.IsPermission(err):
-			c.HandleDockerError(fmt.Sprintf("You do not have permission to connect to the Docker daemon (via %s). This process requires running as the root user.", url.Path))
+			glog.Fatalf("You do not have permission to connect to the Docker daemon (via %s). This process requires running as the root user.", url.Path)
 			return
 		case err == nil && s.IsDir():
-			c.HandleDockerError(fmt.Sprintf("The Docker socket at %s is a directory instead of a unix socket - check that you have configured your connection to the Docker daemon properly.", url.Path))
+			glog.Fatalf("The Docker socket at %s is a directory instead of a unix socket - check that you have configured your connection to the Docker daemon properly.", url.Path)
 			return
 		}
 	}
 	if err := dockerClient.Ping(); err != nil {
-		c.HandleDockerError(fmt.Sprintf("Docker could not be reached at %s.  Docker must be installed and running to start containers.\n%v", dockerAddr, err))
+		glog.Fatalf("Docker could not be reached at %s.  Docker must be installed and running to start containers.\n%v", endpoint, err)
 		return
 	}
 
-	glog.Infof("Connecting to Docker at %s", dockerAddr)
+	glog.Infof("Connecting to Docker at %s", endpoint)
 
 	version, err := dockerClient.Version()
 	if err != nil {
-		c.HandleDockerError(fmt.Sprintf("Unable to check for Docker server version.\n%v", err))
+		glog.Fatalf("Unable to check for Docker server version.\n%v", err)
 		return
 	}
 
 	serverVersion, err := dockerclient.NewAPIVersion(version.APIVersion)
 	if err != nil {
-		c.HandleDockerError(fmt.Sprintf("Unable to determine Docker server version from %q.\n%v", version.APIVersion, err))
+		glog.Fatalf("Unable to determine Docker server version from %q.\n%v", version.APIVersion, err)
 		return
 	}
 
 	minimumPullByIDVersion, err := dockerclient.NewAPIVersion(minimumDockerAPIVersionWithPullByID)
 	if err != nil {
-		c.HandleDockerError(fmt.Sprintf("Unable to check for Docker server version.\n%v", err))
+		glog.Fatalf("Unable to check for Docker server version.\n%v", err)
 		return
 	}
 
 	if serverVersion.LessThan(minimumPullByIDVersion) {
-		c.HandleDockerError(fmt.Sprintf("Docker 1.6 or later (server API version %s or later) required.", minimumDockerAPIVersionWithPullByID))
+		glog.Fatalf("Docker 1.6 or later (server API version %s or later) required.", minimumDockerAPIVersionWithPullByID)
 		return
 	}
 
-	c.DockerClient = dockerClient
-}
-
-// HandleDockerError handles an an error from the docker daemon
-func (c *NodeConfig) HandleDockerError(message string) {
-	if !c.AllowDisabledDocker {
-		glog.Fatalf("error: %s", message)
-	}
-	glog.Errorf("WARNING: %s", message)
-	c.DockerClient = &dockertools.FakeDockerClient{
-		VersionInfo: dockertypes.Version{
-			APIVersion: minimumDockerAPIVersionWithPullByID,
-			Version:    "1.13",
-		},
-		Information: dockertypes.Info{
-			CgroupDriver: "systemd",
-		},
-	}
+	c.DockerClientConfig = dockerClientConfig
 }
 
 // EnsureVolumeDir attempts to convert the provided volume directory argument to
@@ -251,7 +245,7 @@ func (c *NodeConfig) RunKubelet() {
 	}
 
 	// only set when ContainerRuntime == "docker"
-	c.KubeletDeps.DockerClient = c.DockerClient
+	c.KubeletDeps.DockerClientConfig = c.DockerClientConfig
 	// updated by NodeConfig.EnsureVolumeDir
 	c.KubeletServer.RootDirectory = c.VolumeDir
 

pkg/cmd/server/kubernetes/node/node_config.go

@@ -4,6 +4,7 @@ import (
 	"crypto/tls"
 
 	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/kubelet/dockershim"
 
 	kclientsetexternal "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/util/cert"
@@ -12,7 +13,6 @@ import (
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/kubelet"
 	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
-	dockertools "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker"
 	kubeletserver "k8s.io/kubernetes/pkg/kubelet/server"
 
 	configapi "github.com/openshift/origin/pkg/cmd/server/api"
@@ -35,8 +35,8 @@ type NodeConfig struct {
 	// the cluster. It should not be passed into the Kubelet.
 	DNSClient kclientsetexternal.Interface
 
-	// DockerClient is a client to connect to Docker
-	DockerClient dockertools.Interface
+	// DockerClientConfig is a client config to connect to Docker
+	DockerClientConfig *dockershim.ClientConfig
 	// KubeletServer contains the KubeletServer configuration
 	KubeletServer *kubeletoptions.KubeletServer
 	// KubeletDeps are the injected code dependencies for the kubelet, fully initialized

pkg/cmd/server/kubernetes/node/node_config_test.go

@@ -12,12 +12,13 @@ import (
 	"k8s.io/apiserver/pkg/util/flag"
 	kubeproxyoptions "k8s.io/kubernetes/cmd/kube-proxy/app"
 	kubeletoptions "k8s.io/kubernetes/cmd/kubelet/app/options"
-	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/fake"
 	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
+	"k8s.io/kubernetes/pkg/kubelet/config"
 	"k8s.io/kubernetes/pkg/kubelet/rkt"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
+	"k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig"
 )
 
 func TestKubeletDefaults(t *testing.T) {
@@ -29,18 +30,29 @@ func TestKubeletDefaults(t *testing.T) {
 	expectedDefaults := &kubeletoptions.KubeletServer{
 		KubeletFlags: kubeletoptions.KubeletFlags{
 			KubeConfig: flag.NewStringFlag("/var/lib/kubelet/kubeconfig"),
-			ContainerRuntimeOptions: kubeletoptions.ContainerRuntimeOptions{
+			ContainerRuntimeOptions: config.ContainerRuntimeOptions{
 				DockershimRootDirectory:   "/var/lib/dockershim",
-				DockerExecHandlerName:     "native",
 				DockerEndpoint:            "unix:///var/run/docker.sock",
 				ImagePullProgressDeadline: metav1.Duration{Duration: 1 * time.Minute},
 				RktAPIEndpoint:            rkt.DefaultRktAPIServiceEndpoint,
 				PodSandboxImage:           "gcr.io/google_containers/pause-" + goruntime.GOARCH + ":3.0", // overridden
 				DockerDisableSharedPID:    true,
+				ContainerRuntime:          "docker",
 			},
-			CloudProvider: "auto-detect",
-			RootDirectory: "/var/lib/kubelet",
-			CertDirectory: "/var/lib/kubelet/pki",
+			CloudProvider:           "", // now disabled
+			RootDirectory:           "/var/lib/kubelet",
+			CertDirectory:           "/var/lib/kubelet/pki",
+			RegisterNode:            true,                              // this looks suspicious
+			RemoteRuntimeEndpoint:   "unix:///var/run/dockershim.sock", // overridden
+			VolumePluginDir:         "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
+			SeccompProfileRoot:      "/var/lib/kubelet/seccomp",
+			MaxContainerCount:       -1,
+			MasterServiceNamespace:  "default",
+			ExperimentalQOSReserved: map[string]string{},
+			NodeLabels:              map[string]string{},
+			MaxPerPodContainerCount: 1,
+			RegisterSchedulable:     true,
+			NonMasqueradeCIDR:       "10.0.0.0/8",
 		},
 
 		KubeletConfiguration: kubeletconfig.KubeletConfiguration{
@@ -65,18 +77,21 @@ func TestKubeletDefaults(t *testing.T) {
 			VolumeStatsAggPeriod: metav1.Duration{Duration: time.Minute},
 			CgroupRoot:           "",
 			CgroupDriver:         "cgroupfs",
-			ClusterDNS:           nil, // overridden
-			ClusterDomain:        "",  // overridden
-			ContainerRuntime:     "docker",
-			Containerized:        false, // overridden based on OPENSHIFT_CONTAINERIZED
-			CPUCFSQuota:          true,  // forced to true
-
-			EventBurst:                  10,
-			EventRecordQPS:              5.0,
-			EnableCustomMetrics:         false,
-			EnableDebuggingHandlers:     true,
-			EnableServer:                true,
-			EvictionHard:                "memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%",
+			ClusterDNS:           nil,  // overridden
+			ClusterDomain:        "",   // overridden
+			CPUCFSQuota:          true, // forced to true
+
+			EventBurst:     10,
+			EventRecordQPS: 5.0,
+			//EnableCustomMetrics:         false,
+			EnableDebuggingHandlers: true,
+			EnableServer:            true,
+			EvictionHard: map[string]string{
+				"memory.available":  "100Mi",
+				"nodefs.available":  "10%",
+				"nodefs.inodesFree": "5%",
+				"imagefs.available": "15%",
+			},
 			FileCheckFrequency:          metav1.Duration{Duration: 20 * time.Second}, // overridden
 			HealthzBindAddress:          "127.0.0.1",                                 // disabled
 			HealthzPort:                 10248,                                       // disabled
@@ -92,31 +107,19 @@ func TestKubeletDefaults(t *testing.T) {
 			// TODO figure out where this moved
 			// LowDiskSpaceThresholdMB:        0, // used to be 256.  Overriden to have old behavior. 3.7
 			MakeIPTablesUtilChains:    true,
-			MasterServiceNamespace:    "default",
-			MaxContainerCount:         -1,
-			MaxPerPodContainerCount:   1,
 			MaxOpenFiles:              1000000,
 			MaxPods:                   110, // overridden
-			MinimumGCAge:              metav1.Duration{},
-			NonMasqueradeCIDR:         "10.0.0.0/8",
-			VolumePluginDir:           "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
 			NodeStatusUpdateFrequency: metav1.Duration{Duration: 10 * time.Second},
-			NodeLabels:                nil,
 			OOMScoreAdj:               -999,
-			LockFilePath:              "",
 			Port:                      10250, // overridden
 			ReadOnlyPort:              10255, // disabled
-			RegisterNode:              true,
-			RegisterSchedulable:       true,
 			RegistryBurst:             10,
 			RegistryPullQPS:           5.0,
-			RemoteRuntimeEndpoint:     "unix:///var/run/dockershim.sock", // overridden
 			ResolverConfig:            kubetypes.ResolvConfDefault,
 			KubeletCgroups:            "",
 			CgroupsPerQOS:             true,
 			// TODO figure out where this moved
-			// RootDirectory:                  "/var/lib/kubelet", // overridden
-			RuntimeCgroups:                 "",
+			//RuntimeCgroups:                 "",
 			SerializeImagePulls:            true,
 			StreamingConnectionIdleTimeout: metav1.Duration{Duration: 4 * time.Hour},
 			SyncFrequency:                  metav1.Duration{Duration: 1 * time.Minute},
@@ -125,22 +128,15 @@ func TestKubeletDefaults(t *testing.T) {
 			TLSPrivateKeyFile:              "", // overridden to prevent cert generation
 			KubeAPIQPS:                     5.0,
 			KubeAPIBurst:                   10,
-			// TODO figure out where this moved
-			// OutOfDiskTransitionFrequency:   metav1.Duration{Duration: 5 * time.Minute},
-			HairpinMode:        "promiscuous-bridge",
-			SeccompProfileRoot: "/var/lib/kubelet/seccomp",
-			// TODO figure out where this moved
-			// CloudProvider:                "auto-detect",
-			RuntimeRequestTimeout:        metav1.Duration{Duration: 2 * time.Minute},
-			ContentType:                  "application/vnd.kubernetes.protobuf",
-			EnableControllerAttachDetach: true,
-			ExperimentalQOSReserved:      kubeletconfig.ConfigurationMap{},
+			HairpinMode:                    "promiscuous-bridge",
+			RuntimeRequestTimeout:          metav1.Duration{Duration: 2 * time.Minute},
+			ContentType:                    "application/vnd.kubernetes.protobuf",
+			EnableControllerAttachDetach:   true,
 
-			EvictionPressureTransitionPeriod:    metav1.Duration{Duration: 5 * time.Minute},
-			ExperimentalKernelMemcgNotification: false,
+			EvictionPressureTransitionPeriod: metav1.Duration{Duration: 5 * time.Minute},
 
-			SystemReserved: kubeletconfig.ConfigurationMap{},
-			KubeReserved:   kubeletconfig.ConfigurationMap{},
+			SystemReserved: nil,
+			KubeReserved:   nil,
 
 			EnforceNodeAllocatable: []string{"pods"},
 
@@ -151,7 +147,7 @@ func TestKubeletDefaults(t *testing.T) {
 	}
 
 	if goruntime.GOOS == "darwin" {
-		expectedDefaults.KubeletConfiguration.RemoteRuntimeEndpoint = ""
+		//expectedDefaults.KubeletConfiguration.RemoteRuntimeEndpoint = ""
 	}
 
 	if !reflect.DeepEqual(defaults, expectedDefaults) {
@@ -166,33 +162,38 @@ func TestProxyConfig(t *testing.T) {
 	// Once we've reacted to the changes appropriately in buildKubeProxyConfig(), update this expected default to match the new upstream defaults
 	oomScoreAdj := int32(-999)
 	ipTablesMasqueratebit := int32(14)
+	conntrackMin := int32(128 * 1024)
+	conntrackMaxPerCore := int32(32 * 1024)
 
-	expectedProxyConfig := &componentconfig.KubeProxyConfiguration{
+	expectedProxyConfig := &kubeproxyconfig.KubeProxyConfiguration{
 		BindAddress:        "0.0.0.0",
 		HealthzBindAddress: "0.0.0.0:10256",   // disabled
 		MetricsBindAddress: "127.0.0.1:10249", // disabled
-		ClientConnection: componentconfig.ClientConnectionConfiguration{
+		ClientConnection: kubeproxyconfig.ClientConnectionConfiguration{
 			ContentType: "application/vnd.kubernetes.protobuf",
 			QPS:         5,
 			Burst:       10,
 		},
-		IPTables: componentconfig.KubeProxyIPTablesConfiguration{
+		IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{
 			MasqueradeBit: &ipTablesMasqueratebit,
 			SyncPeriod:    metav1.Duration{Duration: 30 * time.Second},
 		},
+		IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{
+			SyncPeriod: metav1.Duration{Duration: 30 * time.Second},
+		},
 		OOMScoreAdj:       &oomScoreAdj,  // disabled
 		ResourceContainer: "/kube-proxy", // disabled
 		UDPIdleTimeout:    metav1.Duration{Duration: 250 * time.Millisecond},
-		Conntrack: componentconfig.KubeProxyConntrackConfiguration{
-			Min:                   128 * 1024,
-			MaxPerCore:            32 * 1024,
-			TCPEstablishedTimeout: metav1.Duration{Duration: 86400 * time.Second}, // 1 day (1/5 default)
-			TCPCloseWaitTimeout:   metav1.Duration{Duration: 1 * time.Hour},
+		Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{
+			Min:                   &conntrackMin,
+			MaxPerCore:            &conntrackMaxPerCore,
+			TCPEstablishedTimeout: &metav1.Duration{Duration: 86400 * time.Second}, // 1 day (1/5 default)
+			TCPCloseWaitTimeout:   &metav1.Duration{Duration: 1 * time.Hour},
 		},
 		ConfigSyncPeriod: metav1.Duration{Duration: 15 * time.Minute},
 	}
 
-	actualDefaultConfig, _ := kubeproxyoptions.NewOptions()
+	actualDefaultConfig := kubeproxyoptions.NewOptions()
 	actualConfig, _ := actualDefaultConfig.ApplyDefaults(actualDefaultConfig.GetConfig())
 
 	if !reflect.DeepEqual(expectedProxyConfig, actualConfig) {

pkg/cmd/server/kubernetes/node/options/options.go

@@ -77,7 +77,6 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, error)
 	server.MaxPods = 250
 	server.PodsPerCore = 10
 	server.CgroupDriver = "systemd"
-	server.DockerExecHandlerName = string(options.DockerConfig.ExecHandlerName)
 	server.RemoteRuntimeEndpoint = options.DockerConfig.DockerShimSocket
 	server.RemoteImageEndpoint = options.DockerConfig.DockerShimSocket
 	server.DockershimRootDirectory = options.DockerConfig.DockershimRootDirectory
@@ -128,7 +127,7 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, error)
 
 	// terminate early if feature gate is incorrect on the node
 	if len(server.FeatureGates) > 0 {
-		if err := utilfeature.DefaultFeatureGate.Set(server.FeatureGates); err != nil {
+		if err := utilfeature.DefaultFeatureGate.SetFromMap(server.FeatureGates); err != nil {
 			return nil, err
 		}
 	}
@@ -143,7 +142,6 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, error)
 	if network.IsOpenShiftNetworkPlugin(options.NetworkConfig.NetworkPluginName) {
 		// SDN plugin pod setup/teardown is implemented as a CNI plugin
 		server.NetworkPluginName = kubeletcni.CNIPluginName
-		server.NetworkPluginDir = kubeletcni.DefaultNetDir
 		server.CNIConfDir = kubeletcni.DefaultNetDir
 		server.CNIBinDir = kubeletcni.DefaultCNIDir
 		server.HairpinMode = kubeletconfig.HairpinNone