Merge pull request #16270 from smarterclayton/exec_kubelet

Automatic merge from submit-queue (batch tested with PRs 16480, 16486, 16270, 16128, 16489)

Direct exec() the kubelet instead of launching in proc

If only the kubelet is launched by the node process, execve(2) instead of launching in process. Requires some corrections to the upstream flags to support round tripping.  Support `OPENSHIFT_ALLOW_UNSUPPORTED_KUBELET=<path>` to allow a kubelet binary that is not exactly equivalent (symlink or hardlink) to the current file.  If the kubelet binary cannot be found, print a warning and continue with the in-proc flow (so we don't break older users without the kubelet symlink).

To start:

```
$ openshift start node --config=... --enable=kubelet --loglevel=3
<will log, then exec kubelet>
... kubelet logs
```

Networking can be run separately with:

```
$ openshift start network --config=...
```

Did a quick sanity test against this, didn't hit any obvious issues.

Builds off #16269

pkg/cmd/flagtypes/glog.go

@@ -14,10 +14,18 @@ func GLog(flags *pflag.FlagSet) {
 		level := flag.Value.(*glog.Level)
 		levelPtr := (*int32)(level)
 		flags.Int32Var(levelPtr, "loglevel", 0, "Set the level of log output (0-10)")
+		if flags.Lookup("v") == nil {
+			flags.Int32Var(levelPtr, "v", 0, "Set the level of log output (0-10)")
+		}
+		flags.Lookup("v").Hidden = true
 	}
 	if flag := from.Lookup("vmodule"); flag != nil {
 		value := flag.Value
 		flags.Var(pflagValue{value}, "logspec", "Set per module logging with file|pattern=LEVEL,...")
+		if flags.Lookup("vmodule") == nil {
+			flags.Var(pflagValue{value}, "vmodule", "Set per module logging with file|pattern=LEVEL,...")
+		}
+		flags.Lookup("vmodule").Hidden = true
 	}
 }
 

pkg/cmd/server/kubernetes/node/options/options.go

@@ -70,8 +70,6 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, *compon
 	server.HTTPCheckFrequency = metav1.Duration{Duration: time.Duration(0)} // no remote HTTP pod creation access
 	server.FileCheckFrequency = metav1.Duration{Duration: time.Duration(fileCheckInterval) * time.Second}
 	server.KubeletFlags.ContainerRuntimeOptions.PodSandboxImage = imageTemplate.ExpandOrDie("pod")
-	server.LowDiskSpaceThresholdMB = 256 // this the previous default
-	server.CPUCFSQuota = true            // enable cpu cfs quota enforcement by default
 	server.MaxPods = 250
 	server.PodsPerCore = 10
 	server.CgroupDriver = "systemd"
@@ -203,3 +201,7 @@ func buildKubeProxyConfig(options configapi.NodeConfig) (*componentconfig.KubePr
 
 	return proxyconfig, nil
 }
+
+func ToFlags(config *kubeletoptions.KubeletServer) []string {
+	return cmdflags.AsArgs(config.AddFlags, kubeletoptions.NewKubeletServer().AddFlags)
+}

pkg/cmd/server/start/start_node.go

@@ -5,15 +5,20 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
+	"syscall"
 
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
 
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	kubeletoptions "k8s.io/kubernetes/cmd/kubelet/app/options"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
 	kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
 
@@ -328,12 +333,85 @@ func (o NodeOptions) IsRunFromConfig() bool {
 	return (len(o.ConfigFile) > 0)
 }
 
+// execKubelet attempts to call execve() for the kubelet with the configuration defined
+// in server passed as flags. If the binary is not the same as the current file and
+// the environment variable OPENSHIFT_ALLOW_UNSUPPORTED_KUBELET is unset, the method
+// will return an error. The returned boolean indicates whether fallback to in-process
+// is allowed.
+func execKubelet(server *kubeletoptions.KubeletServer) (bool, error) {
+	// verify the Kubelet binary to use
+	path := "kubelet"
+	requireSameBinary := true
+	if newPath := os.Getenv("OPENSHIFT_ALLOW_UNSUPPORTED_KUBELET"); len(newPath) > 0 {
+		requireSameBinary = false
+		path = newPath
+	}
+	kubeletPath, err := exec.LookPath(path)
+	if err != nil {
+		return requireSameBinary, err
+	}
+	kubeletFile, err := os.Stat(kubeletPath)
+	if err != nil {
+		return requireSameBinary, err
+	}
+	thisPath, err := exec.LookPath(os.Args[0])
+	if err != nil {
+		return true, err
+	}
+	thisFile, err := os.Stat(thisPath)
+	if err != nil {
+		return true, err
+	}
+	if !os.SameFile(thisFile, kubeletFile) {
+		if requireSameBinary {
+			return true, fmt.Errorf("binary at %q is not the same file as %q, cannot execute", thisPath, kubeletPath)
+		}
+		glog.Warningf("UNSUPPORTED: Executing a different Kubelet than the current binary is not supported: %s", kubeletPath)
+	}
+
+	// convert current settings to flags
+	args := nodeoptions.ToFlags(server)
+	args = append([]string{kubeletPath}, args...)
+	for i := glog.Level(10); i > 0; i-- {
+		if glog.V(i) {
+			args = append(args, fmt.Sprintf("--v=%d", i))
+			break
+		}
+	}
+	for i, s := range os.Args {
+		if s == "--vmodule" {
+			if i+1 < len(os.Args) {
+				args = append(args, fmt.Sprintf("--vmodule=", os.Args[i+1]))
+				break
+			}
+		}
+		if strings.HasPrefix(s, "--vmodule=") {
+			args = append(args, s)
+			break
+		}
+	}
+	glog.V(3).Infof("Exec %s %s", kubeletPath, strings.Join(args, " "))
+	return false, syscall.Exec(kubeletPath, args, os.Environ())
+}
+
 func StartNode(nodeConfig configapi.NodeConfig, components *utilflags.ComponentFlag) error {
 	server, proxyConfig, err := nodeoptions.Build(nodeConfig)
 	if err != nil {
 		return err
 	}
 
+	// as a step towards decomposing OpenShift into Kubernetes components, perform an execve
+	// to launch the Kubelet instead of loading in-process
+	if components.Calculated().Equal(sets.NewString(ComponentKubelet)) {
+		ok, err := execKubelet(server)
+		if !ok {
+			return err
+		}
+		if err != nil {
+			utilruntime.HandleError(fmt.Errorf("Unable to call exec on kubelet, continuing with normal startup: %v", err))
+		}
+	}
+
 	networkConfig, err := network.New(nodeConfig, server.ClusterDomain, proxyConfig, components.Enabled(ComponentProxy), components.Enabled(ComponentDNS) && len(nodeConfig.DNSBindAddress) > 0)
 	if err != nil {
 		return err

pkg/cmd/util/flags/flags.go

@@ -36,6 +36,33 @@ func Resolve(args map[string][]string, fn func(*pflag.FlagSet)) []error {
 	return Apply(args, fs)
 }
 
+func AsArgs(fn, defaultFn func(*pflag.FlagSet)) []string {
+	fs := pflag.NewFlagSet("extended", pflag.ContinueOnError)
+	fn(fs)
+	defaults := pflag.NewFlagSet("defaults", pflag.ContinueOnError)
+	defaultFn(defaults)
+	var args []string
+	fs.VisitAll(func(flag *pflag.Flag) {
+		defaultFlag := defaults.Lookup(flag.Name)
+		s := flag.Value.String()
+		defaultValue := defaultFlag.Value.String()
+		if s == defaultValue {
+			return
+		}
+		if values, err := fs.GetStringSlice(flag.Name); err == nil {
+			for _, s := range values {
+				args = append(args, fmt.Sprintf("--%s=%s", flag.Name, s))
+			}
+		} else {
+			if len(s) == 0 {
+				s = defaultValue
+			}
+			args = append(args, fmt.Sprintf("--%s=%s", flag.Name, s))
+		}
+	})
+	return args
+}
+
 // ComponentFlag represents a set of enabled components used in a command line.
 type ComponentFlag struct {
 	enabled    string