diff --git a/pkg/network/common/egressip.go b/pkg/network/common/egressip.go index edea2db2c5fd..c2da0635c3c7 100644 --- a/pkg/network/common/egressip.go +++ b/pkg/network/common/egressip.go @@ -8,6 +8,7 @@ import ( "github.com/golang/glog" + ktypes "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" @@ -63,6 +64,7 @@ type EgressIPTracker struct { watcher EgressIPWatcher + nodes map[ktypes.UID]*nodeEgress nodesByNodeIP map[string]*nodeEgress namespacesByVNID map[uint32]*namespaceEgress egressIPs map[string]*egressIPInfo @@ -77,6 +79,7 @@ func NewEgressIPTracker(watcher EgressIPWatcher) *EgressIPTracker { return &EgressIPTracker{ watcher: watcher, + nodes: make(map[ktypes.UID]*nodeEgress), nodesByNodeIP: make(map[string]*nodeEgress), namespacesByVNID: make(map[uint32]*namespaceEgress), egressIPs: make(map[string]*egressIPInfo), @@ -167,9 +170,10 @@ func (eit *EgressIPTracker) handleDeleteHostSubnet(obj interface{}) { hs := obj.(*networkapi.HostSubnet) glog.V(5).Infof("Watch %s event for HostSubnet %q", watch.Deleted, hs.Name) - eit.UpdateHostSubnetEgress(&networkapi.HostSubnet{ - HostIP: hs.HostIP, - }) + hs = hs.DeepCopy() + hs.EgressCIDRs = nil + hs.EgressIPs = nil + eit.UpdateHostSubnetEgress(hs) } func (eit *EgressIPTracker) UpdateHostSubnetEgress(hs *networkapi.HostSubnet) { @@ -185,7 +189,7 @@ func (eit *EgressIPTracker) UpdateHostSubnetEgress(hs *networkapi.HostSubnet) { sdnIP = netutils.GenerateDefaultGateway(cidr).String() } - node := eit.nodesByNodeIP[hs.HostIP] + node := eit.nodes[hs.UID] if node == nil { if len(hs.EgressIPs) == 0 && len(hs.EgressCIDRs) == 0 { return @@ -196,9 +200,11 @@ func (eit *EgressIPTracker) UpdateHostSubnetEgress(hs *networkapi.HostSubnet) { sdnIP: sdnIP, requestedIPs: sets.NewString(), } + eit.nodes[hs.UID] = node eit.nodesByNodeIP[hs.HostIP] = node } else if len(hs.EgressIPs) == 0 && len(hs.EgressCIDRs) == 0 { - delete(eit.nodesByNodeIP, hs.HostIP) + delete(eit.nodes, hs.UID) + delete(eit.nodesByNodeIP, node.nodeIP) } // Process EgressCIDRs @@ -218,6 +224,28 @@ func (eit *EgressIPTracker) UpdateHostSubnetEgress(hs *networkapi.HostSubnet) { eit.updateEgressCIDRs = true } + if node.nodeIP != hs.HostIP { + // We have to clean up the old egress IP mappings and call syncEgressIPs + // before we can change node.nodeIP + movedEgressIPs := make([]string, 0, node.requestedIPs.Len()) + for _, ip := range node.requestedIPs.UnsortedList() { + eg := eit.egressIPs[ip] + if eg.assignedNodeIP == node.nodeIP { + movedEgressIPs = append(movedEgressIPs, ip) + eit.deleteNodeEgressIP(node, ip) + } + } + eit.syncEgressIPs() + + delete(eit.nodesByNodeIP, node.nodeIP) + node.nodeIP = hs.HostIP + eit.nodesByNodeIP[node.nodeIP] = node + + for _, ip := range movedEgressIPs { + eit.addNodeEgressIP(node, ip) + } + } + // Process new and removed EgressIPs oldRequestedIPs := node.requestedIPs node.requestedIPs = sets.NewString(hs.EgressIPs...) @@ -448,7 +476,7 @@ func (eit *EgressIPTracker) findEgressIPAllocation(ip net.IP, allocation map[str bestNode := "" otherNodes := false - for _, node := range eit.nodesByNodeIP { + for _, node := range eit.nodes { egressIPs, exists := allocation[node.nodeName] if !exists { continue @@ -478,7 +506,7 @@ func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string { allocation := make(map[string][]string) changed := make(map[string]bool) alreadyAllocated := make(map[string]bool) - for _, node := range eit.nodesByNodeIP { + for _, node := range eit.nodes { if len(node.parsedCIDRs) > 0 { allocation[node.nodeName] = make([]string, 0, node.requestedIPs.Len()) } @@ -534,7 +562,7 @@ func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string { } // Remove unchanged nodes from the return value - for _, node := range eit.nodesByNodeIP { + for _, node := range eit.nodes { if !changed[node.nodeName] { delete(allocation, node.nodeName) } diff --git a/pkg/network/common/egressip_test.go b/pkg/network/common/egressip_test.go index dd23961b0908..db14b5393529 100644 --- a/pkg/network/common/egressip_test.go +++ b/pkg/network/common/egressip_test.go @@ -947,3 +947,57 @@ func TestEgressCIDRAllocation(t *testing.T) { t.Fatalf("%v", err) } } + +func TestEgressNodeRenumbering(t *testing.T) { + eit, w := setupEgressIPTracker(t) + + updateHostSubnetEgress(eit, &networkapi.HostSubnet{ + Host: "alpha", + HostIP: "172.17.0.3", + EgressIPs: []string{"172.17.0.100"}, + }) + updateHostSubnetEgress(eit, &networkapi.HostSubnet{ + Host: "beta", + HostIP: "172.17.0.4", + EgressIPs: []string{"172.17.0.101"}, + }) + updateHostSubnetEgress(eit, &networkapi.HostSubnet{ + Host: "gamma", + HostIP: "172.17.0.5", + EgressIPs: []string{"172.17.0.102"}, + }) + updateNetNamespaceEgress(eit, &networkapi.NetNamespace{ + NetID: 42, + EgressIPs: []string{"172.17.0.100"}, + }) + updateNetNamespaceEgress(eit, &networkapi.NetNamespace{ + NetID: 43, + EgressIPs: []string{"172.17.0.101"}, + }) + + err := w.assertChanges( + "claim 172.17.0.100 on 172.17.0.3 for namespace 42", + "namespace 42 via 172.17.0.100 on 172.17.0.3", + "claim 172.17.0.101 on 172.17.0.4 for namespace 43", + "namespace 43 via 172.17.0.101 on 172.17.0.4", + ) + if err != nil { + t.Fatalf("%v", err) + } + + // Renumber one of the hosts + updateHostSubnetEgress(eit, &networkapi.HostSubnet{ + Host: "beta", + HostIP: "172.17.0.6", + EgressIPs: []string{"172.17.0.101"}, + }) + err = w.assertChanges( + "release 172.17.0.101 on 172.17.0.4", + "namespace 43 dropped", + "claim 172.17.0.101 on 172.17.0.6 for namespace 43", + "namespace 43 via 172.17.0.101 on 172.17.0.6", + ) + if err != nil { + t.Fatalf("%v", err) + } +} diff --git a/pkg/network/node/egressip_test.go b/pkg/network/node/egressip_test.go index 30286987d019..d0656a98c4a6 100644 --- a/pkg/network/node/egressip_test.go +++ b/pkg/network/node/egressip_test.go @@ -722,3 +722,62 @@ func TestMarkForVNID(t *testing.T) { } } } + +func TestEgressNodeRenumbering(t *testing.T) { + eip, flows := setupEgressIPWatcher(t) + + eip.tracker.UpdateHostSubnetEgress(&networkapi.HostSubnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "alpha", + UID: ktypes.UID("alpha"), + }, + Host: "alpha", + HostIP: "172.17.0.3", + EgressIPs: []string{"172.17.0.100"}, + }) + eip.tracker.UpdateHostSubnetEgress(&networkapi.HostSubnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "beta", + UID: ktypes.UID("beta"), + }, + Host: "beta", + HostIP: "172.17.0.4", + EgressIPs: []string{"172.17.0.101"}, + }) + eip.tracker.UpdateHostSubnetEgress(&networkapi.HostSubnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gamma", + UID: ktypes.UID("gamma"), + }, + Host: "gamma", + HostIP: "172.17.0.5", + EgressIPs: []string{"172.17.0.102"}, + }) + updateNamespaceEgress(eip, 42, []string{"172.17.0.100"}) + updateNamespaceEgress(eip, 43, []string{"172.17.0.101"}) + + err := assertOVSChanges(eip, &flows, + egressOVSChange{vnid: 42, egress: Remote, remote: "172.17.0.3"}, + egressOVSChange{vnid: 43, egress: Local}, + ) + if err != nil { + t.Fatalf("%v", err) + } + + // Renumber one of the hosts + eip.tracker.UpdateHostSubnetEgress(&networkapi.HostSubnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "beta", + UID: ktypes.UID("beta"), + }, + Host: "beta", + HostIP: "172.17.0.6", + EgressIPs: []string{"172.17.0.101"}, + }) + err = assertOVSChanges(eip, &flows, + egressOVSChange{vnid: 43, egress: Remote, remote: "172.17.0.6"}, + ) + if err != nil { + t.Fatalf("%v", err) + } +}