[WIP] HIVE-2270: proper e2e support for vSphere

apis/hive/v1/vsphere/platform.go

@@ -1,12 +1,14 @@
 package vsphere
 
 import (
+	configv1 "github.com/openshift/api/config/v1"
 	corev1 "k8s.io/api/core/v1"
 )
 
 // Platform stores any global configuration used for vSphere platforms.
 type Platform struct {
 	// VCenter is the domain name or IP address of the vCenter.
+	// Deprecated: Please use Platform.VSphere instead
 	VCenter string `json:"vCenter"`
 
 	// CredentialsSecretRef refers to a secret that contains the vSphere account access
@@ -18,18 +20,26 @@ type Platform struct {
 	CertificatesSecretRef corev1.LocalObjectReference `json:"certificatesSecretRef"`
 
 	// Datacenter is the name of the datacenter to use in the vCenter.
+	// Deprecated: Please use Platform.VSphere instead
 	Datacenter string `json:"datacenter"`
 
 	// DefaultDatastore is the default datastore to use for provisioning volumes.
+	// Deprecated: Please use Platform.VSphere instead
 	DefaultDatastore string `json:"defaultDatastore"`
 
 	// Folder is the name of the folder that will be used and/or created for
 	// virtual machines.
+	// Deprecated: Please use Platform.VSphere instead
 	Folder string `json:"folder,omitempty"`
 
 	// Cluster is the name of the cluster virtual machines will be cloned into.
+	// Deprecated: Please use Platform.VSphere instead
 	Cluster string `json:"cluster,omitempty"`
 
 	// Network specifies the name of the network to be used by the cluster.
+	// Deprecated: Please use Platform.VSphere instead
 	Network string `json:"network,omitempty"`
+
+	// VSphere is the full spec of the vSphere platform.
+	VSphere *configv1.VSpherePlatformSpec `json:"vSphere,omitempty"`
 }

contrib/pkg/createcluster/create.go

@@ -2,7 +2,9 @@ package createcluster
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
+	vcmv1 "github.com/openshift-splat-team/vsphere-capacity-manager/pkg/apis/vspherecapacitymanager.splat.io/v1"
 	"os"
 	"os/user"
 	"path/filepath"
@@ -189,15 +191,10 @@ type Options struct {
 	OpenStackIngressFloatingIP string
 
 	// VSphere
-	VSphereVCenter          string
-	VSphereDatacenter       string
-	VSphereDefaultDataStore string
-	VSphereFolder           string
-	VSphereCluster          string
-	VSphereAPIVIP           string
-	VSphereIngressVIP       string
-	VSphereNetwork          string
-	VSphereCACerts          string
+	VSphereAPIVIP     string
+	VSphereIngressVIP string
+	VSphereLeases     string
+	VSphereCACerts    string
 
 	// Ovirt
 	OvirtClusterID       string
@@ -345,14 +342,9 @@ OpenShift Installer publishes all the services of the cluster like API server an
 	flags.StringVar(&opt.OpenStackIngressFloatingIP, "openstack-ingress-floating-ip", "", "Floating IP address to use for cluster's Ingress service")
 
 	// vSphere flags
-	flags.StringVar(&opt.VSphereVCenter, "vsphere-vcenter", "", "Domain name or IP address of the vCenter")
-	flags.StringVar(&opt.VSphereDatacenter, "vsphere-datacenter", "", "Datacenter to use in the vCenter")
-	flags.StringVar(&opt.VSphereDefaultDataStore, "vsphere-default-datastore", "", "Default datastore to use for provisioning volumes")
-	flags.StringVar(&opt.VSphereFolder, "vsphere-folder", "", "Folder that will be used and/or created for virtual machines")
-	flags.StringVar(&opt.VSphereCluster, "vsphere-cluster", "", "Cluster virtual machines will be cloned into")
 	flags.StringVar(&opt.VSphereAPIVIP, "vsphere-api-vip", "", "Virtual IP address for the api endpoint")
 	flags.StringVar(&opt.VSphereIngressVIP, "vsphere-ingress-vip", "", "Virtual IP address for ingress application routing")
-	flags.StringVar(&opt.VSphereNetwork, "vsphere-network", "", "Name of the network to be used by the cluster")
+	flags.StringVar(&opt.VSphereLeases, "vsphere-leases", "", "Path to vsphere-capacity-manager lease, multiple lease paths can be : delimited ")
 	flags.StringVar(&opt.VSphereCACerts, "vsphere-ca-certs", "", "Path to vSphere CA certificate, multiple CA paths can be : delimited")
 
 	// oVirt flags
@@ -728,47 +720,28 @@ func (o *Options) GenerateObjects() ([]runtime.Object, error) {
 			caCerts = append(caCerts, caCert)
 		}
 
-		vSphereNetwork := os.Getenv(constants.VSphereNetworkEnvVar)
-		if o.VSphereNetwork != "" {
-			vSphereNetwork = o.VSphereNetwork
-		}
-
-		vSphereDatacenter := os.Getenv(constants.VSphereDataCenterEnvVar)
-		if o.VSphereDatacenter != "" {
-			vSphereDatacenter = o.VSphereDatacenter
-		}
-		if vSphereDatacenter == "" {
-			return nil, fmt.Errorf("must provide --vsphere-datacenter or set %s env var", constants.VSphereDataCenterEnvVar)
-		}
-
-		vSphereDatastore := os.Getenv(constants.VSphereDataStoreEnvVar)
-		if o.VSphereDefaultDataStore != "" {
-			vSphereDatastore = o.VSphereDefaultDataStore
-		}
-		if vSphereDatastore == "" {
-			return nil, fmt.Errorf("must provide --vsphere-default-datastore or set %s env var", constants.VSphereDataStoreEnvVar)
-		}
-
-		vSphereVCenter := os.Getenv(constants.VSphereVCenterEnvVar)
-		if o.VSphereVCenter != "" {
-			vSphereVCenter = o.VSphereVCenter
+		vcmLeases := []*vcmv1.LeaseStatus{}
+		for _, leasePath := range filepath.SplitList(o.VSphereLeases) {
+			leaseBytes, err := os.ReadFile(leasePath)
+			if err != nil {
+				return nil, fmt.Errorf("error reading %s: %w", leasePath, err)
+			}
+			lease := vcmv1.LeaseStatus{}
+			err = json.Unmarshal(leaseBytes, &lease)
+			if err != nil {
+				return nil, fmt.Errorf("error decoding lease %s: %w", leasePath, err)
+			}
+			vcmLeases = append(vcmLeases, &lease)
 		}
-		if vSphereVCenter == "" {
-			return nil, fmt.Errorf("must provide --vsphere-vcenter or set %s env var", constants.VSphereVCenterEnvVar)
+		if len(vcmLeases) == 0 {
+			return nil, fmt.Errorf("must provide at least one vsphere lease path with --vsphere-leases")
 		}
 
 		vsphereProvider := &clusterresource.VSphereCloudBuilder{
-			VCenter:          vSphereVCenter,
-			Username:         vsphereUsername,
-			Password:         vspherePassword,
-			Datacenter:       vSphereDatacenter,
-			DefaultDatastore: vSphereDatastore,
-			Folder:           o.VSphereFolder,
-			Cluster:          o.VSphereCluster,
-			APIVIP:           o.VSphereAPIVIP,
-			IngressVIP:       o.VSphereIngressVIP,
-			Network:          vSphereNetwork,
-			CACert:           bytes.Join(caCerts, []byte("\n")),
+			Username: vsphereUsername,
+			Password: vspherePassword,
+			CACert:   bytes.Join(caCerts, []byte("\n")),
+			VSphere:  clusterresource.VSphereSpecFromVCMLeasesAndIPs(vcmLeases, o.VSphereAPIVIP, o.VSphereIngressVIP),
 		}
 		builder.CloudBuilder = vsphereProvider
 	case cloudOVirt:

go.mod

@@ -131,6 +131,7 @@ require (
 
 require (
 	github.com/gophercloud/utils/v2 v2.0.0-20240701101423-2401526caee5
+	github.com/openshift-splat-team/vsphere-capacity-manager v0.0.0-20250108203126-b0e32b706ec3
 	gopkg.in/evanphx/json-patch.v4 v4.12.0
 )
 
@@ -355,7 +356,7 @@ require (
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go.uber.org/automaxprocs v1.5.3 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.26.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/sync v0.10.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect
 	golang.org/x/term v0.27.0 // indirect

go.sum

@@ -1168,6 +1168,8 @@ github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQ
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/openshift-splat-team/vsphere-capacity-manager v0.0.0-20250108203126-b0e32b706ec3 h1:1p5XISWXTdNjcEmWv4B3O74kdBdHMP4f1TVch5cpjss=
+github.com/openshift-splat-team/vsphere-capacity-manager v0.0.0-20250108203126-b0e32b706ec3/go.mod h1:ZduYDGQUoRafxEZgN5MrMsE0Ne4FtMgXV3d8fnrfV/0=
 github.com/openshift/api v0.0.0-20210730095913-85e1d547cdee/go.mod h1:ntkQrC1Z6AxxkhDlVpDVjkD+pzdwVUalWyfH40rSyyM=
 github.com/openshift/api v0.0.0-20210816181336-8ff39b776da3/go.mod h1:x81TFA31x1OMT9SYWukQqJ/KbmeveRN6fo+XeGRK8g0=
 github.com/openshift/api v0.0.0-20240919193929-2669d1ebc910 h1:8pA9Ugq0hhUbVaVWi5lgF4PGaV1ozpTI1NYn/QWqspg=
@@ -1624,8 +1626,8 @@ go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
 go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
 go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
-go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
-go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

hack/e2e-common.sh

@@ -222,24 +222,30 @@ case "${CLOUD}" in
 	BASE_DOMAIN="${BASE_DOMAIN:-origin-ci-int-gce.dev.openshift.com}"
 	;;
 "vsphere")
-  BASE_DOMAIN="${BASE_DOMAIN:-vmc.devcluster.openshift.com}"
-  if [ -z "$NETWORK_NAME" ]; then
-    echo "Variable 'NETWORK_NAME' not set."
+  if [ -z "$SHARED_DIR" ]; then
+    echo "Variable 'SHARED_DIR' not set."
+    exit 1
+  fi
+  if [ -z "$ADDITIONAL_CLUSTER_API_VIP" ]; then
+    echo "Variable 'ADDITIONAL_CLUSTER_API_VIP' not set."
     exit 1
   fi
-   if [ -z "$VCENTER" ]; then
-    echo "Variable 'VCENTER' not set."
+  if [ -z "$ADDITIONAL_CLUSTER_INGRESS_VIP" ]; then
+    echo "Variable 'ADDITIONAL_CLUSTER_INGRESS_VIP' not set."
     exit 1
   fi
-  API_VIP=$(get_vips 3) # Get 3rd vip from file
-  INGRESS_VIP=$(get_vips 4) # Get 4th vip from file
-  EXTRA_CREATE_CLUSTER_ARGS="--vsphere-datacenter=${GOVC_DATACENTER:-DEVQEdatacenter} \
-      --vsphere-default-datastore=${GOVC_DATASTORE:-vsanDatastore}\
-      --vsphere-cluster=${VSPHERE_CLUSTER:-DEVQEcluster}
-      --vsphere-api-vip=$API_VIP \
-      --vsphere-ingress-vip=$INGRESS_VIP \
-      --vsphere-network=$NETWORK_NAME \
-      --vsphere-vcenter=$VCENTER"
+  if [ -z "$ADDITIONAL_CLUSTER_NAME" ]; then
+    echo "Variable 'ADDITIONAL_CLUSTER_NAME' not set."
+    exit 1
+  fi
+  VSPHERE_LEASES=$(find "$SHARED_DIR" -name "LEASE_*.json" | grep -v "LEASE_single.json" | paste -sd ":" -)
+  BASE_DOMAIN=$(<"${SHARED_DIR}"/basedomain.txt)
+  BASE_DOMAIN="${BASE_DOMAIN:-vmc.devcluster.openshift.com}"
+  CLUSTER_NAME="${ADDITIONAL_CLUSTER_NAME}"
+
+  EXTRA_CREATE_CLUSTER_ARGS="--vsphere-leases=$VSPHERE_LEASES \
+      --vsphere-api-vip=$ADDITIONAL_CLUSTER_API_VIP \
+      --vsphere-ingress-vip=$ADDITIONAL_CLUSTER_INGRESS_VIP"
   ;;
 *)
 	echo "unknown cloud: ${CLOUD}"
@@ -318,21 +324,3 @@ function capture_cluster_logs() {
         exit 1
     fi
 }
-
-function get_vips() {
-  # Return vip at given index
-  idx=${1:-1} 
-   if [ -z "$SHARED_DIR" ]; then
-    echo "Variable 'SHARED_DIR' not set."
-    exit 1
-  fi
-
-  vips="${SHARED_DIR}/vips.txt"
-  if [ ! -f "$vips" ]; then
-    echo "Error: File '$vips' not found."
-    exit 1
-  fi
-
-  vip=$(sed -n "${idx}p" "$vips")
-  echo "$vip"
-}