From d62cd1ac8532833de7da58a1ba0253c191a60af2 Mon Sep 17 00:00:00 2001 From: devtools-bot Date: Mon, 17 Feb 2025 16:30:43 +0800 Subject: [PATCH] check load balancer healthcheck port and path --- test/extended/cloud_controller_manager/ccm.go | 52 ++++++++++++ .../generated/zz_generated.annotations.go | 2 + test/extended/util/aws_client.go | 82 +++++++++++++++++++ test/extended/util/framework.go | 23 ++++++ 4 files changed, 159 insertions(+) create mode 100644 test/extended/util/aws_client.go diff --git a/test/extended/cloud_controller_manager/ccm.go b/test/extended/cloud_controller_manager/ccm.go index 42a028b275c3..cc7f3606ea0b 100644 --- a/test/extended/cloud_controller_manager/ccm.go +++ b/test/extended/cloud_controller_manager/ccm.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "fmt" + "strings" + "time" "github.com/ghodss/yaml" g "github.com/onsi/ginkgo/v2" @@ -12,6 +14,8 @@ import ( exutil "github.com/openshift/origin/test/extended/util" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" ) const cloudControllerNamespace = "openshift-cloud-controller-manager" @@ -88,6 +92,38 @@ var _ = g.Describe("[sig-cloud-provider][Feature:OpenShiftCloudControllerManager o.HaveField("Contents", o.ContainSubstring("cloud-provider=external")), ))) }) + + g.It("Cluster scoped load balancer healthcheck port and path should be 10256/healthz", func() { + exutil.SkipIfNotPlatform(oc, "AWS") + if strings.HasPrefix(exutil.GetClusterRegion(oc), "us-iso") { + g.Skip("Skipped: There is no public subnet on AWS C2S/SC2S disconnected clusters!") + } + + g.By("Create a cluster scope load balancer") + svcName := "test-lb" + defer oc.WithoutNamespace().AsAdmin().Run("delete").Args("-n", oc.Namespace(), "service", "loadbalancer", svcName, "--ignore-not-found").Execute() + out, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-n", oc.Namespace(), "service", "loadbalancer", svcName, "--tcp=80:8080").Output() + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create lb service") + o.Expect(out).To(o.ContainSubstring("service/" + svcName + " created")) + + g.By("Check External-IP assigned") + svcExternalIP := getLoadBalancerExternalIP(oc, oc.Namespace(), svcName) + e2e.Logf("External IP assigned: %s", svcExternalIP) + o.Expect(svcExternalIP).NotTo(o.BeEmpty(), "externalIP should not be empty") + lbName := strings.Split(svcExternalIP, "-")[0] + + g.By("Check healthcheck port and path should be 10256/healthz") + healthCheckPort := "10256" + healthCheckPath := "/healthz" + exutil.GetAwsCredentialFromCluster(oc) + region := exutil.GetClusterRegion(oc) + sess := exutil.InitAwsSession(region) + elbClient := exutil.NewELBClient(sess) + healthCheck, err := elbClient.GetLBHealthCheckPortPath(lbName) + o.Expect(err).NotTo(o.HaveOccurred(), "unable to get health check port and path") + e2e.Logf("Health check port and path: %v", healthCheck) + o.Expect(healthCheck).To(o.Equal(fmt.Sprintf("HTTP:%s%s", healthCheckPort, healthCheckPath))) + }) }) // isPlatformExternal returns true when the platform has an in-tree provider, @@ -103,3 +139,19 @@ func isPlatformExternal(platformType configv1.PlatformType) bool { return false } } + +// getLoadBalancerExternalIP get IP address of LB service +func getLoadBalancerExternalIP(oc *exutil.CLI, namespace string, svcName string) string { + var svcExternalIP string + var cmdErr error + checkErr := wait.Poll(5*time.Second, 300*time.Second, func() (bool, error) { + svcExternalIP, cmdErr = oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.status.loadBalancer.ingress[0].hostname}").Output() + if svcExternalIP == "" || cmdErr != nil { + e2e.Logf("Waiting for lb service IP assignment. Trying again...") + return false, nil + } + return true, nil + }) + o.Expect(checkErr).NotTo(o.HaveOccurred()) + return svcExternalIP +} diff --git a/test/extended/util/annotate/generated/zz_generated.annotations.go b/test/extended/util/annotate/generated/zz_generated.annotations.go index aa6c753f2461..a60e565472e7 100644 --- a/test/extended/util/annotate/generated/zz_generated.annotations.go +++ b/test/extended/util/annotate/generated/zz_generated.annotations.go @@ -961,6 +961,8 @@ var Annotations = map[string]string{ "[sig-cli][Slow] can use rsync to upload files to pods [apigroup:template.openshift.io] using a watch should watch for changes and rsync them": "", + "[sig-cloud-provider][Feature:OpenShiftCloudControllerManager][Late] Cluster scoped load balancer healthcheck port and path should be 10256/healthz": " [Suite:openshift/conformance/parallel]", + "[sig-cloud-provider][Feature:OpenShiftCloudControllerManager][Late] Deploy an external cloud provider [apigroup:machineconfiguration.openshift.io]": " [Suite:openshift/conformance/parallel]", "[sig-cluster-lifecycle] CSRs from machines that are not recognized by the cloud provider are not approved": " [Suite:openshift/conformance/parallel]", diff --git a/test/extended/util/aws_client.go b/test/extended/util/aws_client.go new file mode 100644 index 000000000000..1072175268c5 --- /dev/null +++ b/test/extended/util/aws_client.go @@ -0,0 +1,82 @@ +package util + +import ( + "encoding/base64" + "os" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/elb" + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + "github.com/tidwall/gjson" + + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +// GetAwsCredentialFromCluster get aws credential from cluster +func GetAwsCredentialFromCluster(oc *CLI) { + credential, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("secret/aws-creds", "-n", "kube-system", "-o", "json").Output() + // Skip for sts and c2s clusters. + if err != nil { + g.Skip("Did not get credential to access aws, skip the testing.") + } + o.Expect(err).NotTo(o.HaveOccurred()) + accessKeyIDBase64, secureKeyBase64 := gjson.Get(credential, `data.aws_access_key_id`).String(), gjson.Get(credential, `data.aws_secret_access_key`).String() + accessKeyID, err1 := base64.StdEncoding.DecodeString(accessKeyIDBase64) + o.Expect(err1).NotTo(o.HaveOccurred()) + secureKey, err2 := base64.StdEncoding.DecodeString(secureKeyBase64) + o.Expect(err2).NotTo(o.HaveOccurred()) + clusterRegion, err3 := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.aws.region}").Output() + o.Expect(err3).NotTo(o.HaveOccurred()) + os.Setenv("AWS_ACCESS_KEY_ID", string(accessKeyID)) + os.Setenv("AWS_SECRET_ACCESS_KEY", string(secureKey)) + os.Setenv("AWS_REGION", clusterRegion) +} + +// InitAwsSession init session +func InitAwsSession(region string) *session.Session { + sess := session.Must(session.NewSessionWithOptions(session.Options{ + Config: aws.Config{ + Region: aws.String(region), + }, + })) + + return sess +} + +type ELBClient struct { + svc *elb.ELB +} + +// NewELBClient creates an ECRClient +func NewELBClient(sess *session.Session) *ELBClient { + return &ELBClient{ + svc: elb.New(sess), + } +} + +// GetLBHealthCheckPortPath get load balance health check port and path +func (elbClient *ELBClient) GetLBHealthCheckPortPath(lbName string) (string, error) { + input := &elb.DescribeLoadBalancersInput{ + LoadBalancerNames: []*string{ + aws.String(lbName), + }, + } + + result, err := elbClient.svc.DescribeLoadBalancers(input) + if err != nil { + e2e.Logf("Failed to describe load balancer: %v", err) + return "", err + } + + if len(result.LoadBalancerDescriptions) == 0 { + e2e.Logf("Failed to get load balancers: %v", err) + } + + healthCheck := result.LoadBalancerDescriptions[0].HealthCheck + if healthCheck == nil { + e2e.Logf("Failed to get health check: %v", err) + } + return *healthCheck.Target, nil +} diff --git a/test/extended/util/framework.go b/test/extended/util/framework.go index 69cdcc64b717..cf6bbf4af1e6 100644 --- a/test/extended/util/framework.go +++ b/test/extended/util/framework.go @@ -2403,3 +2403,26 @@ func IsCapabilityEnabled(oc *CLI, cap configv1.ClusterVersionCapability) (bool, } return false, nil } + +// SkipIfNotPlatform skip the test if supported platforms are not matched +func SkipIfNotPlatform(oc *CLI, platforms ...configv1.PlatformType) { + var match bool + infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + for _, platform := range platforms { + if infra.Status.PlatformStatus.Type == platform { + match = true + break + } + } + if !match { + g.Skip("Skip this test scenario because it is not supported on the " + string(infra.Status.PlatformStatus.Type) + " platform") + } +} + +// GetClusterRegion get the cluster's region +func GetClusterRegion(oc *CLI) string { + region, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node", `-ojsonpath={.items[].metadata.labels.topology\.kubernetes\.io/region}`).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + return region +}