sdn: convert pod network setup to a CNI plugin

Use the same kubelet network plugin interfaces as everyone else.  This converts the
openshift-sdn plugin from a plugin compiled into kubelet to one that uses the
kubelet CNI driver to call a standard CNI plugin.  This plugin sends requests
from kubelet over a root-only unix domain socket back to the openshift-sdn
node process which handles the actual pod setup/teardown operations.

We want to consolidate these operations inside the node process instead of
leaving them to the CNI plugin itself because we need to ensure serialized
access to OVS, and we need a long-running process to handle HostPort reservation.
While we could serialize operations for each pod rather than serializing
all pod operations, it turns out to be difficult and error-prone to ensure
previous operations complete and those operations can still be GCed in a
race-safe manner.

General flow:

1) kubelet wants to set up pod networking
2) kubelet calls internal CNI driver
3) CNI driver looks for CNI network config files, finds
/etc/cni/net.d/80-openshift-sdn.conf, and calls the
/opt/cni/bin/openshift-sdn CNI plugin executable with CNI_COMMAND=ADD
4) openshift-sdn CNI plugin sends environment and stdin to the
openshift-node process via HTTP over a root-only unix domain socket
5) openshift-node process sets up pod networking with OVS, veth
creation
6) openshift-node process calls the CNI 'host-local' IPAM plugin to
allocate an IP address for the pod from the local node subnet
7) openshift-node process returns the IPAM details via HTTP over the
unix domain socket to the waiting openshift-sdn CNI plugin
8) openshift-sdn CNI plugin prints IPAM details to stdout
9) kubelet reads IPAM details (or error) and completes pod setup

contrib/node/install-sdn.sh

@@ -1,25 +1,30 @@
 #!/bin/bash
 
 os::provision::install-sdn() {
-  local default_target="/usr"
-
   local deployed_root=$1
-  local target=${2:-${default_target}}
+  local binaries_path=$2
+  local target=${3:-}
+  local target_usrdir="${target}/usr"
+  local target_bindir="${target_usrdir}/bin"
+  local target_confdir="${target}/etc/cni/net.d"
+  local target_cnidir="${target}/opt/cni/bin"
 
-  if [ ! -d ${target} ]; then
-    mkdir -p ${target}
-  fi
+  mkdir -p -m u+rwx,g+rwx,o+rx "${target_usrdir}"
+  mkdir -p -m u+rwx,g+rwx,o+rx "${target_bindir}"
+  mkdir -p -m u+rwx,g+rwx,o+rx "${target_confdir}"
+  mkdir -p -m u+rwx,g+rwx,o+rx "${target_cnidir}"
 
   local osdn_plugin_path="${deployed_root}/pkg/sdn/plugin"
-  mkdir -p "${target}/bin/"
-  pushd "${osdn_plugin_path}" > /dev/null
-    install bin/openshift-sdn-ovs "${target}/bin/"
-  popd > /dev/null
+  install -m u+rwx,g+rwx,o+rx "${osdn_plugin_path}/bin/openshift-sdn-ovs" "${target_bindir}"
+  install -m u+rw,g+rw,o+r "${osdn_plugin_path}/sdn-cni-plugin/80-openshift-sdn.conf" "${target_confdir}"
+
+  install -m u+rwx,g+rwx,o+rx "${binaries_path}/sdn-cni-plugin" "${target_cnidir}/openshift-sdn"
+  install -m u+rwx,g+rwx,o+rx "${binaries_path}/host-local" "${target_cnidir}"
+  install -m u+rwx,g+rwx,o+rx "${binaries_path}/loopback" "${target_cnidir}"
 
-  # Assume a non-default target is an indication of deploying in an
-  # environment where openvswitch is managed in a separate container
-  # (e.g. atomic host).
-  if [[ "${target}" = "${default_target}" ]]; then
+  # Assume an empty/default target is an indication of deploying in an
+  # environment where openvswitch should be started by us
+  if [[ -z "${target}" ]]; then
     systemctl enable openvswitch
     systemctl start openvswitch
   fi

contrib/vagrant/provision-util.sh

@@ -49,7 +49,7 @@ os::provision::base-install() {
 
   echo "Installing openshift"
   os::provision::install-cmds "${origin_root}"
-  os::provision::install-sdn "${origin_root}"
+  os::provision::install-sdn "${origin_root}" "$(os::build::get-bin-output-path "${OS_ROOT}")"
   os::provision::set-os-env "${origin_root}" "${config_root}"
 }
 

hack/build-go.sh

@@ -4,11 +4,13 @@
 STARTTIME=$(date +%s)
 source "$(dirname "${BASH_SOURCE}")/lib/init.sh"
 
+build_targets=("$@")
+platform="$(os::build::host_platform)"
+
 # only works on Linux for now, all other platforms must build binaries themselves
 if [[ -z "$@" ]]; then
   if [[ "${OS_RELEASE:-}" != "n" ]] && \
     os::build::detect_local_release_tars $(os::build::host_platform_friendly) >/dev/null; then
-    platform=$(os::build::host_platform)
     echo "++ Using release artifacts from ${OS_RELEASE_COMMIT} for ${platform} instead of building"
     mkdir -p "${OS_OUTPUT_BINPATH}/${platform}"
     os::build::extract_tar "${OS_PRIMARY_RELEASE_TAR}" "${OS_OUTPUT_BINPATH}/${platform}"
@@ -19,10 +21,17 @@ if [[ -z "$@" ]]; then
 
     ret=$?; ENDTIME=$(date +%s); echo "$0 took $(($ENDTIME - $STARTTIME)) seconds"; exit "$ret"
   fi
+
+  build_targets=("${OS_ALL_TARGETS[@]}")
+  # Also build SDN components on Linux by default
+  if [[ "${platform}" =~ linux/.* ]]; then
+    build_targets=("${build_targets[@]}" "${OS_SDN_COMPILE_TARGETS_LINUX[@]}")
+  fi
 fi
 
-os::build::build_binaries "$@"
-os::build::place_bins "$@"
+OS_BUILD_PLATFORMS=("${OS_BUILD_PLATFORMS[@]:-${platform}}")
+os::build::build_binaries "${build_targets[@]}"
+os::build::place_bins "${build_targets[@]}"
 os::build::make_openshift_binary_symlinks
 
 ret=$?; ENDTIME=$(date +%s); echo "$0 took $(($ENDTIME - $STARTTIME)) seconds"; exit "$ret"

hack/build-images.sh

@@ -77,7 +77,7 @@ ln_or_cp "${imagedir}/oc"              examples/gitserver/bin
 ln_or_cp "${imagedir}/dockerregistry"  images/dockerregistry/bin
 
 # Copy SDN scripts into images/node
-os::provision::install-sdn "${OS_ROOT}" "${OS_ROOT}/images/node"
+os::provision::install-sdn "${OS_ROOT}" "${imagedir}" "${OS_ROOT}/images/node"
 mkdir -p images/node/conf/
 cp -pf "${OS_ROOT}/contrib/systemd/openshift-sdn-ovs.conf" images/node/conf/
 

hack/common.sh

@@ -23,10 +23,16 @@ readonly -f os::build::host_platform
 
 readonly OS_IMAGE_COMPILE_PLATFORMS=("$(os::build::host_platform)")
 
+readonly OS_SDN_COMPILE_TARGETS_LINUX=(
+  pkg/sdn/plugin/sdn-cni-plugin
+  vendor/github.com/containernetworking/cni/plugins/ipam/host-local
+  vendor/github.com/containernetworking/cni/plugins/main/loopback
+)
 readonly OS_IMAGE_COMPILE_TARGETS=(
   images/pod
   cmd/dockerregistry
   cmd/gitserver
+  "${OS_SDN_COMPILE_TARGETS_LINUX[@]}"
 )
 readonly OS_IMAGE_COMPILE_GOFLAGS="-tags include_gcs"
 readonly OS_SCRATCH_IMAGE_COMPILE_TARGETS=(
@@ -356,14 +362,16 @@ function os::build::export_targets() {
   done
 
   if [[ ${#targets[@]} -eq 0 ]]; then
-    targets=("${OS_ALL_TARGETS[@]}")
+    echo "No targets to export!"
+    exit 1
   fi
 
   binaries=($(os::build::binaries_from_targets "${targets[@]}"))
 
   platforms=("${OS_BUILD_PLATFORMS[@]:+${OS_BUILD_PLATFORMS[@]}}")
   if [[ ${#platforms[@]} -eq 0 ]]; then
-    platforms=("$(os::build::host_platform)")
+    echo "No platforms to build for!"
+    exit 1
   fi
 }
 readonly -f os::build::export_targets

hack/dind-cluster.sh

@@ -201,8 +201,11 @@ function copy-runtime() {
 
   cp "$(os::build::find-binary openshift)" "${target}"
   cp "$(os::build::find-binary host-local)" "${target}"
-  local osdn_plugin_path="${origin_root}/pkg/sdn/plugin/bin"
-  cp "${osdn_plugin_path}/openshift-sdn-ovs" "${target}"
+  cp "$(os::build::find-binary loopback)" "${target}"
+  cp "$(os::build::find-binary sdn-cni-plugin)" "${target}/openshift-sdn"
+  local osdn_plugin_path="${origin_root}/pkg/sdn/plugin"
+  cp "${osdn_plugin_path}/bin/openshift-sdn-ovs" "${target}"
+  cp "${osdn_plugin_path}/sdn-cni-plugin/80-openshift-sdn.conf" "${target}"
 }
 
 function wait-for-cluster() {

images/dind/node/Dockerfile

@@ -47,6 +47,7 @@ COPY openshift-enable-ssh-access.service /etc/systemd/system/
 RUN systemctl enable openshift-enable-ssh-access.service
 
 # SDN plugin setup
+RUN mkdir -p /etc/cni/net.d
 RUN mkdir -p /opt/cni/bin
 
 # Symlink from the data path intended to be mounted as a volume to
@@ -63,6 +64,9 @@ RUN ln -sf /data/openshift-sdn-ovs /usr/local/bin/ && \
     ln -sf /data/openshift /usr/local/bin/openshift-docker-build && \
     ln -sf /data/openshift /usr/local/bin/openshift-sti-build && \
     ln -sf /data/openshift /usr/local/bin/openshift-f5-router && \
-    ln -sf /data/host-local /opt/cni/bin/
+    ln -sf /data/openshift-sdn /opt/cni/bin/ && \
+    ln -sf /data/host-local    /opt/cni/bin/ && \
+    ln -sf /data/loopback      /opt/cni/bin/ && \
+    ln -sf /data/80-openshift-sdn.conf /etc/cni/net.d/
 
 ENV KUBECONFIG /data/openshift.local.config/master/admin.kubeconfig

images/node/Dockerfile

@@ -11,6 +11,7 @@
 FROM openshift/origin
 
 COPY bin/* /usr/bin/
+COPY opt/cni/bin/* /opt/cni/bin/
 COPY conf/openshift-sdn-ovs.conf /usr/lib/systemd/system/origin-node.service.d/
 COPY scripts/* /usr/local/bin/
 

origin.spec

@@ -264,9 +264,16 @@ mkdir -p %{buildroot}%{_sharedstatedir}/origin
 
 
 # Install sdn scripts
-pushd pkg/sdn/plugin/bin
+install -d -m 0755 %{buildroot}%{_sysconfdir}/cni/net.d
+pushd pkg/sdn/plugin/sdn-cni-plugin
+   install -p -m 644 80-openshift-sdn.conf %{buildroot}%{_sysconfdir}/cni/net.d
    install -p -m 755 openshift-sdn-ovs %{buildroot}%{_bindir}/openshift-sdn-ovs
 popd
+install -d -m 0755 %{buildroot}/opt/cni/bin
+install -p -m 0755 _build/bin/sdn-cni-plugin %{buildroot}/opt/cni/bin/openshift-sdn
+install -p -m 0755 _build/bin/host-local %{buildroot}/opt/cni/bin
+install -p -m 0755 _build/bin/loopback %{buildroot}/opt/cni/bin
+
 install -d -m 0755 %{buildroot}%{_unitdir}/%{name}-node.service.d
 install -p -m 0644 contrib/systemd/openshift-sdn-ovs.conf %{buildroot}%{_unitdir}/%{name}-node.service.d/openshift-sdn-ovs.conf
 
@@ -411,8 +418,12 @@ fi
 %files sdn-ovs
 %dir %{_unitdir}/docker.service.d/
 %dir %{_unitdir}/%{name}-node.service.d/
+%dir %{_sysconfdir}/cni/net.d
+%dir /opt/cni/bin
 %{_bindir}/openshift-sdn-ovs
 %{_unitdir}/%{name}-node.service.d/openshift-sdn-ovs.conf
+%{_sysconfdir}/cni/net.d/80-openshift-sdn.conf
+/opt/cni/bin/*
 
 %posttrans sdn-ovs
 # This path was installed by older packages but the directory wasn't owned by

pkg/cmd/server/kubernetes/node_config.go

@@ -22,6 +22,8 @@ import (
 	clientadapter "k8s.io/kubernetes/pkg/client/unversioned/adapters/internalclientset"
 	"k8s.io/kubernetes/pkg/kubelet"
 	"k8s.io/kubernetes/pkg/kubelet/dockertools"
+	kubeletnetwork "k8s.io/kubernetes/pkg/kubelet/network"
+	kubeletcni "k8s.io/kubernetes/pkg/kubelet/network/cni"
 	kubeletserver "k8s.io/kubernetes/pkg/kubelet/server"
 	kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
 	kcrypto "k8s.io/kubernetes/pkg/util/crypto"
@@ -190,11 +192,35 @@ func BuildKubernetesNodeConfig(options configapi.NodeConfig, enableProxy, enable
 		return nil, err
 	}
 
+	// Initialize SDN before building kubelet config so it can modify options
+	iptablesSyncPeriod, err := time.ParseDuration(options.IPTablesSyncPeriod)
+	if err != nil {
+		return nil, fmt.Errorf("Cannot parse the provided ip-tables sync period (%s) : %v", options.IPTablesSyncPeriod, err)
+	}
+	sdnPlugin, err := sdnplugin.NewNodePlugin(options.NetworkConfig.NetworkPluginName, originClient, kubeClient, options.NodeName, options.NodeIP, iptablesSyncPeriod, options.NetworkConfig.MTU)
+	if err != nil {
+		return nil, fmt.Errorf("SDN initialization failed: %v", err)
+	}
+	if sdnPlugin != nil {
+		// SDN plugin pod setup/teardown is implemented as a CNI plugin
+		server.NetworkPluginName = kubeletcni.CNIPluginName
+		server.NetworkPluginDir = kubeletcni.DefaultNetDir
+		server.HairpinMode = componentconfig.HairpinNone
+		server.ConfigureCBR0 = false
+	}
+
 	deps, err := kubeletapp.UnsecuredKubeletDeps(server)
 	if err != nil {
 		return nil, err
 	}
 
+	// Replace the kubelet-created CNI plugin with the SDN plugin
+	// Kubelet must be initialized with NetworkPluginName="cni" but
+	// the SDN plugin (if available) needs to be the only one used
+	if sdnPlugin != nil {
+		deps.NetworkPlugins = []kubeletnetwork.NetworkPlugin{sdnPlugin}
+	}
+
 	// provide any config overrides
 	//deps.NodeName = options.NodeName
 	deps.KubeClient = clientadapter.FromUnversionedClient(kubeClient)
@@ -250,18 +276,6 @@ func BuildKubernetesNodeConfig(options configapi.NodeConfig, enableProxy, enable
 		deps.TLSOptions = nil
 	}
 
-	iptablesSyncPeriod, err := time.ParseDuration(options.IPTablesSyncPeriod)
-	if err != nil {
-		return nil, fmt.Errorf("Cannot parse the provided ip-tables sync period (%s) : %v", options.IPTablesSyncPeriod, err)
-	}
-	sdnPlugin, err := sdnplugin.NewNodePlugin(options.NetworkConfig.NetworkPluginName, originClient, kubeClient, options.NodeName, options.NodeIP, iptablesSyncPeriod, options.NetworkConfig.MTU)
-	if err != nil {
-		return nil, fmt.Errorf("SDN initialization failed: %v", err)
-	}
-	if sdnPlugin != nil {
-		deps.NetworkPlugins = append(deps.NetworkPlugins, sdnPlugin)
-	}
-
 	sdnProxy, err := sdnplugin.NewProxyPlugin(options.NetworkConfig.NetworkPluginName, originClient, kubeClient)
 	if err != nil {
 		return nil, fmt.Errorf("SDN proxy initialization failed: %v", err)