Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

OCPQE-28543: Check load balancer healthcheck port and path on aws #29544

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/MakeNowJust/heredoc v1.0.0
github.com/RangelReale/osincli v0.0.0-20160924135400-fababb0555f2
github.com/apparentlymart/go-cidr v1.1.0
github.com/aws/aws-sdk-go v1.44.204
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/distribution/distribution/v3 v3.0.0-20230530204932-ba46c769b3d1
github.com/fsouza/go-dockerclient v1.12.0
Expand Down Expand Up @@ -47,6 +48,7 @@ require (
github.com/spf13/viper v1.8.1
github.com/stretchr/objx v0.5.2
github.com/stretchr/testify v1.9.0
github.com/tidwall/gjson v1.18.0
go.etcd.io/etcd/client/pkg/v3 v3.5.16
go.etcd.io/etcd/client/v3 v3.5.16
golang.org/x/crypto v0.31.0
Expand Down Expand Up @@ -183,6 +185,7 @@ require (
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jonboulle/clockwork v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
Expand Down Expand Up @@ -241,6 +244,8 @@ require (
github.com/src-d/gcfg v1.4.0 // indirect
github.com/stoewer/go-strcase v1.3.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xanzy/ssh-agent v0.2.1 // indirect
Expand Down
15 changes: 15 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go v1.44.204 h1:7/tPUXfNOHB390A63t6fJIwmlwVQAkAwcbzKsU2/6OQ=
github.com/aws/aws-sdk-go v1.44.204/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
Expand Down Expand Up @@ -461,6 +463,10 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
Expand Down Expand Up @@ -754,6 +760,12 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
Expand Down Expand Up @@ -944,6 +956,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
Expand Down Expand Up @@ -1040,6 +1053,7 @@ golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
Expand All @@ -1054,6 +1068,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
Expand Down
52 changes: 52 additions & 0 deletions test/extended/cloud_controller_manager/ccm.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"time"

"github.com/ghodss/yaml"
g "github.com/onsi/ginkgo/v2"
Expand All @@ -12,6 +14,8 @@ import (
exutil "github.com/openshift/origin/test/extended/util"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/wait"
e2e "k8s.io/kubernetes/test/e2e/framework"
)

const cloudControllerNamespace = "openshift-cloud-controller-manager"
Expand Down Expand Up @@ -88,6 +92,38 @@ var _ = g.Describe("[sig-cloud-provider][Feature:OpenShiftCloudControllerManager
o.HaveField("Contents", o.ContainSubstring("cloud-provider=external")),
)))
})

g.It("Cluster scoped load balancer healthcheck port and path should be 10256/healthz", func() {
Copy link
Contributor

@dgoodwin dgoodwin Feb 19, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The PR looks good, just want to check if that port could ever change, if so you'll save yourself some headaches in having to go submit a test rename PR if you take the port name itself out of the test name and keep it more generic.

"should be correct" or something like that

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The port number is the "kube-proxy" health check port. We don't actually use kube-proxy, but instead OVN-K exposes a health check endpoint at the same port.

If we were to change the port, this would mean all CCMs would have to be updated with a new port, and all implementations/distros of Kube would also need to update

In short, I don't see the port number changing, but, we could change the name anyway just in case

exutil.SkipIfNotPlatform(oc, "AWS")
if strings.HasPrefix(exutil.GetClusterRegion(oc), "us-iso") {
g.Skip("Skipped: There is no public subnet on AWS C2S/SC2S disconnected clusters!")
}

g.By("Create a cluster scope load balancer")
svcName := "test-lb"
defer oc.WithoutNamespace().AsAdmin().Run("delete").Args("-n", oc.Namespace(), "service", "loadbalancer", svcName, "--ignore-not-found").Execute()
out, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", oc.Namespace(), "service", "loadbalancer", svcName, "--tcp=80:8080").Output()
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create lb service")
o.Expect(out).To(o.ContainSubstring("service/" + svcName + " created"))

g.By("Check External-IP assigned")
svcExternalIP := getLoadBalancerExternalIP(oc, oc.Namespace(), svcName)
e2e.Logf("External IP assigned: %s", svcExternalIP)
o.Expect(svcExternalIP).NotTo(o.BeEmpty(), "externalIP should not be empty")
lbName := strings.Split(svcExternalIP, "-")[0]

g.By("Check healthcheck port and path should be 10256/healthz")
healthCheckPort := "10256"
healthCheckPath := "/healthz"
exutil.GetAwsCredentialFromCluster(oc)
region := exutil.GetClusterRegion(oc)
sess := exutil.InitAwsSession(region)
elbClient := exutil.NewELBClient(sess)
healthCheck, err := elbClient.GetLBHealthCheckPortPath(lbName)
o.Expect(err).NotTo(o.HaveOccurred(), "unable to get health check port and path")
e2e.Logf("Health check port and path: %v", healthCheck)
o.Expect(healthCheck).To(o.Equal(fmt.Sprintf("HTTP:%s%s", healthCheckPort, healthCheckPath)))
})
})

// isPlatformExternal returns true when the platform has an in-tree provider,
Expand All @@ -103,3 +139,19 @@ func isPlatformExternal(platformType configv1.PlatformType) bool {
return false
}
}

// getLoadBalancerExternalIP get IP address of LB service
func getLoadBalancerExternalIP(oc *exutil.CLI, namespace string, svcName string) string {
var svcExternalIP string
var cmdErr error
checkErr := wait.Poll(5*time.Second, 300*time.Second, func() (bool, error) {
svcExternalIP, cmdErr = oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.status.loadBalancer.ingress[0].hostname}").Output()
if svcExternalIP == "" || cmdErr != nil {
e2e.Logf("Waiting for lb service IP assignment. Trying again...")
return false, nil
}
return true, nil
})
o.Expect(checkErr).NotTo(o.HaveOccurred())
return svcExternalIP
}

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

82 changes: 82 additions & 0 deletions test/extended/util/aws_client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package util

import (
"encoding/base64"
"os"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/elb"
g "github.com/onsi/ginkgo/v2"
o "github.com/onsi/gomega"
"github.com/tidwall/gjson"

e2e "k8s.io/kubernetes/test/e2e/framework"
)

// GetAwsCredentialFromCluster get aws credential from cluster
func GetAwsCredentialFromCluster(oc *CLI) {
credential, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("secret/aws-creds", "-n", "kube-system", "-o", "json").Output()
// Skip for sts and c2s clusters.
if err != nil {
g.Skip("Did not get credential to access aws, skip the testing.")
}
o.Expect(err).NotTo(o.HaveOccurred())
accessKeyIDBase64, secureKeyBase64 := gjson.Get(credential, `data.aws_access_key_id`).String(), gjson.Get(credential, `data.aws_secret_access_key`).String()
accessKeyID, err1 := base64.StdEncoding.DecodeString(accessKeyIDBase64)
o.Expect(err1).NotTo(o.HaveOccurred())
secureKey, err2 := base64.StdEncoding.DecodeString(secureKeyBase64)
o.Expect(err2).NotTo(o.HaveOccurred())
clusterRegion, err3 := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.aws.region}").Output()
o.Expect(err3).NotTo(o.HaveOccurred())
os.Setenv("AWS_ACCESS_KEY_ID", string(accessKeyID))
os.Setenv("AWS_SECRET_ACCESS_KEY", string(secureKey))
os.Setenv("AWS_REGION", clusterRegion)
}

// InitAwsSession init session
func InitAwsSession(region string) *session.Session {
sess := session.Must(session.NewSessionWithOptions(session.Options{
Config: aws.Config{
Region: aws.String(region),
},
}))

return sess
}

type ELBClient struct {
svc *elb.ELB
}

// NewELBClient creates an ECRClient
func NewELBClient(sess *session.Session) *ELBClient {
return &ELBClient{
svc: elb.New(sess),
}
}

// GetLBHealthCheckPortPath get load balance health check port and path
func (elbClient *ELBClient) GetLBHealthCheckPortPath(lbName string) (string, error) {
input := &elb.DescribeLoadBalancersInput{
LoadBalancerNames: []*string{
aws.String(lbName),
},
}

result, err := elbClient.svc.DescribeLoadBalancers(input)
if err != nil {
e2e.Logf("Failed to describe load balancer: %v", err)
return "", err
}

if len(result.LoadBalancerDescriptions) == 0 {
e2e.Logf("Failed to get load balancers: %v", err)
}

healthCheck := result.LoadBalancerDescriptions[0].HealthCheck
if healthCheck == nil {
e2e.Logf("Failed to get health check: %v", err)
}
return *healthCheck.Target, nil
}
23 changes: 23 additions & 0 deletions test/extended/util/framework.go
Original file line number Diff line number Diff line change
Expand Up @@ -2403,3 +2403,26 @@ func IsCapabilityEnabled(oc *CLI, cap configv1.ClusterVersionCapability) (bool,
}
return false, nil
}

// SkipIfNotPlatform skip the test if supported platforms are not matched
func SkipIfNotPlatform(oc *CLI, platforms ...configv1.PlatformType) {
var match bool
infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
o.Expect(err).NotTo(o.HaveOccurred())
for _, platform := range platforms {
if infra.Status.PlatformStatus.Type == platform {
match = true
break
}
}
if !match {
g.Skip("Skip this test scenario because it is not supported on the " + string(infra.Status.PlatformStatus.Type) + " platform")
}
}

// GetClusterRegion get the cluster's region
func GetClusterRegion(oc *CLI) string {
region, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node", `-ojsonpath={.items[].metadata.labels.topology\.kubernetes\.io/region}`).Output()
o.Expect(err).NotTo(o.HaveOccurred())
return region
}
Loading