Fix SDN exponential backoff timeouts

pkg/network/node/runtime.go

@@ -23,7 +23,7 @@ func (node *OsdnNode) getRuntimeService() (kubeletapi.RuntimeService, error) {
 		kwait.Backoff{
 			Duration: 100 * time.Millisecond,
 			Factor:   1.2,
-			Steps:    23,
+			Steps:    24,
 		},
 		func() (bool, error) {
 			runtimeService, err := kubeletremote.NewRemoteRuntimeService(node.runtimeEndpoint, node.runtimeRequestTimeout)

pkg/network/node/sdn_controller.go

@@ -13,51 +13,13 @@ import (
 	"github.com/openshift/origin/pkg/network/common"
 	"github.com/openshift/origin/pkg/util/netutils"
 
-	kapierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilwait "k8s.io/apimachinery/pkg/util/wait"
 	kapi "k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/util/sysctl"
 
 	"github.com/vishvananda/netlink"
 )
 
-func (plugin *OsdnNode) getLocalSubnet() (string, error) {
-	var subnet *networkapi.HostSubnet
-	// If the HostSubnet doesn't already exist, it will be created by the SDN master in
-	// response to the kubelet registering itself with the master (which should be
-	// happening in another goroutine in parallel with this). Sometimes this takes
-	// unexpectedly long though, so give it plenty of time before returning an error
-	// (since that will cause the node process to exit).
-	backoff := utilwait.Backoff{
-		// A bit over 1 minute total
-		Duration: time.Second,
-		Factor:   1.5,
-		Steps:    8,
-	}
-	err := utilwait.ExponentialBackoff(backoff, func() (bool, error) {
-		var err error
-		subnet, err = plugin.networkClient.Network().HostSubnets().Get(plugin.hostName, metav1.GetOptions{})
-		if err == nil {
-			return true, nil
-		} else if kapierrors.IsNotFound(err) {
-			glog.Warningf("Could not find an allocated subnet for node: %s, Waiting...", plugin.hostName)
-			return false, nil
-		} else {
-			return false, err
-		}
-	})
-	if err != nil {
-		return "", fmt.Errorf("failed to get subnet for this host: %s, error: %v", plugin.hostName, err)
-	}
-
-	if err = plugin.networkInfo.ValidateNodeIP(subnet.HostIP); err != nil {
-		return "", fmt.Errorf("failed to validate own HostSubnet: %v", err)
-	}
-
-	return subnet.Subnet, nil
-}
-
 func (plugin *OsdnNode) alreadySetUp(localSubnetGatewayCIDR string, clusterNetworkCIDR []string) bool {
 	var found bool
 
@@ -106,10 +68,11 @@ func (plugin *OsdnNode) alreadySetUp(localSubnetGatewayCIDR string, clusterNetwo
 }
 
 func deleteLocalSubnetRoute(device, localSubnetCIDR string) {
+	// ~1 sec total
 	backoff := utilwait.Backoff{
 		Duration: 100 * time.Millisecond,
 		Factor:   1.25,
-		Steps:    6,
+		Steps:    7,
 	}
 	err := utilwait.ExponentialBackoff(backoff, func() (bool, error) {
 		l, err := netlink.LinkByName(device)

pkg/network/node/subnets.go

@@ -3,8 +3,13 @@
 package node
 
 import (
+	"fmt"
+	"time"
+
 	"github.com/golang/glog"
 
+	kapierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilwait "k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/tools/cache"
 
@@ -19,14 +24,50 @@ func (node *OsdnNode) SubnetStartNode() error {
 
 type hostSubnetMap map[string]*networkapi.HostSubnet
 
-func (plugin *OsdnNode) updateVXLANMulticastRules(subnets hostSubnetMap) {
+func (node *OsdnNode) getLocalSubnet() (string, error) {
+	var subnet *networkapi.HostSubnet
+	// If the HostSubnet doesn't already exist, it will be created by the SDN master in
+	// response to the kubelet registering itself with the master (which should be
+	// happening in another goroutine in parallel with this). Sometimes this takes
+	// unexpectedly long though, so give it plenty of time before returning an error
+	// (since that will cause the node process to exit).
+	backoff := utilwait.Backoff{
+		// ~2 mins total
+		Duration: time.Second,
+		Factor:   1.5,
+		Steps:    11,
+	}
+	err := utilwait.ExponentialBackoff(backoff, func() (bool, error) {
+		var err error
+		subnet, err = node.networkClient.Network().HostSubnets().Get(node.hostName, metav1.GetOptions{})
+		if err == nil {
+			return true, nil
+		} else if kapierrors.IsNotFound(err) {
+			glog.Warningf("Could not find an allocated subnet for node: %s, Waiting...", node.hostName)
+			return false, nil
+		} else {
+			return false, err
+		}
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to get subnet for this host: %s, error: %v", node.hostName, err)
+	}
+
+	if err = node.networkInfo.ValidateNodeIP(subnet.HostIP); err != nil {
+		return "", fmt.Errorf("failed to validate own HostSubnet: %v", err)
+	}
+
+	return subnet.Subnet, nil
+}
+
+func (node *OsdnNode) updateVXLANMulticastRules(subnets hostSubnetMap) {
 	remoteIPs := make([]string, 0, len(subnets))
 	for _, subnet := range subnets {
-		if subnet.HostIP != plugin.localIP {
+		if subnet.HostIP != node.localIP {
 			remoteIPs = append(remoteIPs, subnet.HostIP)
 		}
 	}
-	if err := plugin.oc.UpdateVXLANMulticastFlows(remoteIPs); err != nil {
+	if err := node.oc.UpdateVXLANMulticastFlows(remoteIPs); err != nil {
 		glog.Errorf("Error updating OVS VXLAN multicast flows: %v", err)
 	}
 }

pkg/oc/admin/diagnostics/diagnostics/network/run_pod.go

@@ -143,7 +143,7 @@ func (d *NetworkDiagnostic) runNetworkDiagnostic() {
 		return
 	}
 	// Wait for network diagnostic pod completion (timeout: ~3 mins)
-	backoff := wait.Backoff{Steps: 38, Duration: 500 * time.Millisecond, Factor: 1.1}
+	backoff := wait.Backoff{Steps: 39, Duration: 500 * time.Millisecond, Factor: 1.1}
 	if err := d.waitForNetworkPod(d.nsName1, util.NetworkDiagPodNamePrefix, backoff, []kapi.PodPhase{kapi.PodSucceeded, kapi.PodFailed}); err != nil {
 		d.res.Error("DNet2007", err, err.Error())
 		return
@@ -164,7 +164,7 @@ func (d *NetworkDiagnostic) runNetworkDiagnostic() {
 	}
 
 	// Wait for network diagnostic pod to start (timeout: ~5 mins)
-	backoff = wait.Backoff{Steps: 36, Duration: time.Second, Factor: 1.1}
+	backoff = wait.Backoff{Steps: 37, Duration: time.Second, Factor: 1.1}
 	if err := d.waitForNetworkPod(d.nsName1, util.NetworkDiagPodNamePrefix, backoff, []kapi.PodPhase{kapi.PodRunning, kapi.PodFailed, kapi.PodSucceeded}); err != nil {
 		d.res.Error("DNet2010", err, err.Error())
 		// Do not bail out here, collect what ever info is available from all valid nodes

pkg/oc/admin/diagnostics/diagnostics/network/setup.go

@@ -155,7 +155,7 @@ func (d *NetworkDiagnostic) waitForTestPodAndService(nsList []string) error {
 	errList := []error{}
 	validPhases := []kapi.PodPhase{kapi.PodRunning, kapi.PodSucceeded, kapi.PodFailed}
 	for _, name := range nsList {
-		backoff := wait.Backoff{Steps: 36, Duration: time.Second, Factor: 1.1} // timeout: ~5 mins
+		backoff := wait.Backoff{Steps: 37, Duration: time.Second, Factor: 1.1} // timeout: ~5 mins
 		if err := d.waitForNetworkPod(name, util.NetworkDiagTestPodNamePrefix, backoff, validPhases); err != nil {
 			errList = append(errList, err)
 		}