Skip to content

Commit

Permalink
Make egress IP code handle node IP address changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@danwinship
danwinship committed Jul 23, 2018
1 parent 4815e18 commit b057704
Show file tree
Hide file tree
Showing 3 changed files with 149 additions and 8 deletions.
44 changes: 36 additions & 8 deletions pkg/network/common/egressip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"github.com/golang/glog"

ktypes "k8s.io/apimachinery/pkg/types"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/watch"
Expand Down Expand Up @@ -63,6 +64,7 @@ type EgressIPTracker struct {

watcher EgressIPWatcher

nodes map[ktypes.UID]*nodeEgress
nodesByNodeIP map[string]*nodeEgress
namespacesByVNID map[uint32]*namespaceEgress
egressIPs map[string]*egressIPInfo
Expand All @@ -77,6 +79,7 @@ func NewEgressIPTracker(watcher EgressIPWatcher) *EgressIPTracker {
return &EgressIPTracker{
watcher: watcher,

nodes: make(map[ktypes.UID]*nodeEgress),
nodesByNodeIP: make(map[string]*nodeEgress),
namespacesByVNID: make(map[uint32]*namespaceEgress),
egressIPs: make(map[string]*egressIPInfo),
Expand Down Expand Up @@ -167,9 +170,10 @@ func (eit *EgressIPTracker) handleDeleteHostSubnet(obj interface{}) {
hs := obj.(*networkapi.HostSubnet)
glog.V(5).Infof("Watch %s event for HostSubnet %q", watch.Deleted, hs.Name)

eit.UpdateHostSubnetEgress(&networkapi.HostSubnet{
HostIP: hs.HostIP,
})
hs = hs.DeepCopy()
hs.EgressCIDRs = nil
hs.EgressIPs = nil
eit.UpdateHostSubnetEgress(hs)
}

func (eit *EgressIPTracker) UpdateHostSubnetEgress(hs *networkapi.HostSubnet) {
Expand All @@ -185,7 +189,7 @@ func (eit *EgressIPTracker) UpdateHostSubnetEgress(hs *networkapi.HostSubnet) {
sdnIP = netutils.GenerateDefaultGateway(cidr).String()
}

node := eit.nodesByNodeIP[hs.HostIP]
node := eit.nodes[hs.UID]
if node == nil {
if len(hs.EgressIPs) == 0 && len(hs.EgressCIDRs) == 0 {
return
Expand All @@ -196,9 +200,11 @@ func (eit *EgressIPTracker) UpdateHostSubnetEgress(hs *networkapi.HostSubnet) {
sdnIP: sdnIP,
requestedIPs: sets.NewString(),
}
eit.nodes[hs.UID] = node
eit.nodesByNodeIP[hs.HostIP] = node
} else if len(hs.EgressIPs) == 0 && len(hs.EgressCIDRs) == 0 {
delete(eit.nodesByNodeIP, hs.HostIP)
delete(eit.nodes, hs.UID)
delete(eit.nodesByNodeIP, node.nodeIP)
}

// Process EgressCIDRs
Expand All @@ -218,6 +224,28 @@ func (eit *EgressIPTracker) UpdateHostSubnetEgress(hs *networkapi.HostSubnet) {
eit.updateEgressCIDRs = true
}

if node.nodeIP != hs.HostIP {
// We have to clean up the old egress IP mappings and call syncEgressIPs
// before we can change node.nodeIP
movedEgressIPs := make([]string, 0, node.requestedIPs.Len())
for _, ip := range node.requestedIPs.UnsortedList() {
eg := eit.egressIPs[ip]
if eg.assignedNodeIP == node.nodeIP {
movedEgressIPs = append(movedEgressIPs, ip)
eit.deleteNodeEgressIP(node, ip)
}
}
eit.syncEgressIPs()

delete(eit.nodesByNodeIP, node.nodeIP)
node.nodeIP = hs.HostIP
eit.nodesByNodeIP[node.nodeIP] = node

for _, ip := range movedEgressIPs {
eit.addNodeEgressIP(node, ip)
}
}

// Process new and removed EgressIPs
oldRequestedIPs := node.requestedIPs
node.requestedIPs = sets.NewString(hs.EgressIPs...)
Expand Down Expand Up @@ -448,7 +476,7 @@ func (eit *EgressIPTracker) findEgressIPAllocation(ip net.IP, allocation map[str
bestNode := ""
otherNodes := false

for _, node := range eit.nodesByNodeIP {
for _, node := range eit.nodes {
egressIPs, exists := allocation[node.nodeName]
if !exists {
continue
Expand Down Expand Up @@ -478,7 +506,7 @@ func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string {
allocation := make(map[string][]string)
changed := make(map[string]bool)
alreadyAllocated := make(map[string]bool)
for _, node := range eit.nodesByNodeIP {
for _, node := range eit.nodes {
if len(node.parsedCIDRs) > 0 {
allocation[node.nodeName] = make([]string, 0, node.requestedIPs.Len())
}
Expand Down Expand Up @@ -534,7 +562,7 @@ func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string {
}

// Remove unchanged nodes from the return value
for _, node := range eit.nodesByNodeIP {
for _, node := range eit.nodes {
if !changed[node.nodeName] {
delete(allocation, node.nodeName)
}
Expand Down
54 changes: 54 additions & 0 deletions pkg/network/common/egressip_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -947,3 +947,57 @@ func TestEgressCIDRAllocation(t *testing.T) {
t.Fatalf("%v", err)
}
}

func TestEgressNodeRenumbering(t *testing.T) {
eit, w := setupEgressIPTracker(t)

updateHostSubnetEgress(eit, &networkapi.HostSubnet{
Host: "alpha",
HostIP: "172.17.0.3",
EgressIPs: []string{"172.17.0.100"},
})
updateHostSubnetEgress(eit, &networkapi.HostSubnet{
Host: "beta",
HostIP: "172.17.0.4",
EgressIPs: []string{"172.17.0.101"},
})
updateHostSubnetEgress(eit, &networkapi.HostSubnet{
Host: "gamma",
HostIP: "172.17.0.5",
EgressIPs: []string{"172.17.0.102"},
})
updateNetNamespaceEgress(eit, &networkapi.NetNamespace{
NetID: 42,
EgressIPs: []string{"172.17.0.100"},
})
updateNetNamespaceEgress(eit, &networkapi.NetNamespace{
NetID: 43,
EgressIPs: []string{"172.17.0.101"},
})

err := w.assertChanges(
"claim 172.17.0.100 on 172.17.0.3 for namespace 42",
"namespace 42 via 172.17.0.100 on 172.17.0.3",
"claim 172.17.0.101 on 172.17.0.4 for namespace 43",
"namespace 43 via 172.17.0.101 on 172.17.0.4",
)
if err != nil {
t.Fatalf("%v", err)
}

// Renumber one of the hosts
updateHostSubnetEgress(eit, &networkapi.HostSubnet{
Host: "beta",
HostIP: "172.17.0.6",
EgressIPs: []string{"172.17.0.101"},
})
err = w.assertChanges(
"release 172.17.0.101 on 172.17.0.4",
"namespace 43 dropped",
"claim 172.17.0.101 on 172.17.0.6 for namespace 43",
"namespace 43 via 172.17.0.101 on 172.17.0.6",
)
if err != nil {
t.Fatalf("%v", err)
}
}
59 changes: 59 additions & 0 deletions pkg/network/node/egressip_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -722,3 +722,62 @@ func TestMarkForVNID(t *testing.T) {
}
}
}

func TestEgressNodeRenumbering(t *testing.T) {
eip, flows := setupEgressIPWatcher(t)

eip.tracker.UpdateHostSubnetEgress(&networkapi.HostSubnet{
ObjectMeta: metav1.ObjectMeta{
Name: "alpha",
UID: ktypes.UID("alpha"),
},
Host: "alpha",
HostIP: "172.17.0.3",
EgressIPs: []string{"172.17.0.100"},
})
eip.tracker.UpdateHostSubnetEgress(&networkapi.HostSubnet{
ObjectMeta: metav1.ObjectMeta{
Name: "beta",
UID: ktypes.UID("beta"),
},
Host: "beta",
HostIP: "172.17.0.4",
EgressIPs: []string{"172.17.0.101"},
})
eip.tracker.UpdateHostSubnetEgress(&networkapi.HostSubnet{
ObjectMeta: metav1.ObjectMeta{
Name: "gamma",
UID: ktypes.UID("gamma"),
},
Host: "gamma",
HostIP: "172.17.0.5",
EgressIPs: []string{"172.17.0.102"},
})
updateNamespaceEgress(eip, 42, []string{"172.17.0.100"})
updateNamespaceEgress(eip, 43, []string{"172.17.0.101"})

err := assertOVSChanges(eip, &flows,
egressOVSChange{vnid: 42, egress: Remote, remote: "172.17.0.3"},
egressOVSChange{vnid: 43, egress: Local},
)
if err != nil {
t.Fatalf("%v", err)
}

// Renumber one of the hosts
eip.tracker.UpdateHostSubnetEgress(&networkapi.HostSubnet{
ObjectMeta: metav1.ObjectMeta{
Name: "beta",
UID: ktypes.UID("beta"),
},
Host: "beta",
HostIP: "172.17.0.6",
EgressIPs: []string{"172.17.0.101"},
})
err = assertOVSChanges(eip, &flows,
egressOVSChange{vnid: 43, egress: Remote, remote: "172.17.0.6"},
)
if err != nil {
t.Fatalf("%v", err)
}
}

0 comments on commit b057704

Please sign in to comment.