From 2c213e9891ef54307e4521782b5dbba3848a5a69 Mon Sep 17 00:00:00 2001 From: Mark Old Date: Thu, 1 May 2025 14:46:51 -0700 Subject: [PATCH] HIVE-2270: minimal e2e support for vSphere --- contrib/pkg/createcluster/create.go | 54 +++++++++++++++++++++++------ hack/e2e-common.sh | 42 +++++----------------- pkg/constants/constants.go | 3 ++ 3 files changed, 56 insertions(+), 43 deletions(-) diff --git a/contrib/pkg/createcluster/create.go b/contrib/pkg/createcluster/create.go index c015ff6947b..e236815499d 100644 --- a/contrib/pkg/createcluster/create.go +++ b/contrib/pkg/createcluster/create.go @@ -2,6 +2,7 @@ package createcluster import ( "bytes" + "encoding/json" "fmt" "os" "os/user" @@ -32,6 +33,7 @@ import ( "github.com/openshift/hive/pkg/gcpclient" "github.com/openshift/hive/pkg/util/scheme" installertypes "github.com/openshift/installer/pkg/types" + installervsphere "github.com/openshift/installer/pkg/types/vsphere" "github.com/openshift/installer/pkg/validate" ) @@ -208,6 +210,7 @@ type Options struct { VSphereAPIVIP string VSphereIngressVIP string VSphereNetwork string + VSpherePlatformSpecJSON string VSphereCACerts string // Ovirt @@ -380,6 +383,7 @@ OpenShift Installer publishes all the services of the cluster like API server an flags.StringVar(&opt.VSphereAPIVIP, "vsphere-api-vip", "", "Virtual IP address for the api endpoint") flags.StringVar(&opt.VSphereIngressVIP, "vsphere-ingress-vip", "", "Virtual IP address for ingress application routing") flags.StringVar(&opt.VSphereNetwork, "vsphere-network", "", "Name of the network to be used by the cluster") + flags.StringVar(&opt.VSpherePlatformSpecJSON, "vsphere-platform-spec-json", "", "Installer vsphere platform spec, encoded as JSON") flags.StringVar(&opt.VSphereCACerts, "vsphere-ca-certs", "", "Path to vSphere CA certificate, multiple CA paths can be : delimited") // Nutanix @@ -786,22 +790,52 @@ func (o *Options) GenerateObjects() ([]runtime.Object, error) { if o.VSphereDatacenter != "" { vSphereDatacenter = o.VSphereDatacenter } - if vSphereDatacenter == "" { - return nil, fmt.Errorf("must provide --vsphere-datacenter or set %s env var", constants.VSphereDataCenterEnvVar) - } vSphereDatastore := os.Getenv(constants.VSphereDataStoreEnvVar) if o.VSphereDefaultDataStore != "" { vSphereDatastore = o.VSphereDefaultDataStore } - if vSphereDatastore == "" { - return nil, fmt.Errorf("must provide --vsphere-default-datastore or set %s env var", constants.VSphereDataStoreEnvVar) - } vSphereVCenter := os.Getenv(constants.VSphereVCenterEnvVar) if o.VSphereVCenter != "" { vSphereVCenter = o.VSphereVCenter } + + vSphereFolder := o.VSphereFolder + vSphereCluster := o.VSphereCluster + vSphereAPIVIP := o.VSphereAPIVIP + vSphereIngressVIP := o.VSphereIngressVIP + + platformBytes := []byte(os.Getenv(constants.VSpherePlatformSpecJSONEnvVar)) + if o.VSpherePlatformSpecJSON != "" { + platformBytes = []byte(o.VSpherePlatformSpecJSON) + } + + if len(platformBytes) > 0 { + o.log.Info("using provided installer platform spec instead of other flags for vsphere (size: %v)", len(platformBytes)) + platform := installervsphere.Platform{} + err = json.Unmarshal(platformBytes, &platform) + if err != nil { + return nil, fmt.Errorf("error decoding platform %s: %w", o.VSpherePlatformSpecJSON, err) + } + + vSphereVCenter = platform.VCenters[0].Server + vSphereDatacenter = platform.VCenters[0].Datacenters[0] + if vSphereDatacenter == "" { + vSphereDatacenter = platform.FailureDomains[0].Topology.Datacenter + } + vSphereDatastore = platform.FailureDomains[0].Topology.Datastore + vSphereFolder = platform.FailureDomains[0].Topology.Folder + vSphereCluster = platform.FailureDomains[0].Topology.ComputeCluster + vSphereNetwork = platform.FailureDomains[0].Topology.Networks[0] + } + + if vSphereDatacenter == "" { + return nil, fmt.Errorf("must provide --vsphere-datacenter or set %s env var", constants.VSphereDataCenterEnvVar) + } + if vSphereDatastore == "" { + return nil, fmt.Errorf("must provide --vsphere-default-datastore or set %s env var", constants.VSphereDataStoreEnvVar) + } if vSphereVCenter == "" { return nil, fmt.Errorf("must provide --vsphere-vcenter or set %s env var", constants.VSphereVCenterEnvVar) } @@ -812,10 +846,10 @@ func (o *Options) GenerateObjects() ([]runtime.Object, error) { Password: vspherePassword, Datacenter: vSphereDatacenter, DefaultDatastore: vSphereDatastore, - Folder: o.VSphereFolder, - Cluster: o.VSphereCluster, - APIVIP: o.VSphereAPIVIP, - IngressVIP: o.VSphereIngressVIP, + Folder: vSphereFolder, + Cluster: vSphereCluster, + APIVIP: vSphereAPIVIP, + IngressVIP: vSphereIngressVIP, Network: vSphereNetwork, CACert: bytes.Join(caCerts, []byte("\n")), } diff --git a/hack/e2e-common.sh b/hack/e2e-common.sh index 7ed11b6c895..54ce60f7fd9 100755 --- a/hack/e2e-common.sh +++ b/hack/e2e-common.sh @@ -3,6 +3,9 @@ shopt -s expand_aliases alias echo='/bin/echo -n `date -Ins --universal`" "; /bin/echo' +# Workaround for upstream vsphere e2e setting SSL_CERT_FILE to a vsphere specific value (bad for us) +unset SSL_CERT_FILE + ### # TEMPORARY workaround for https://issues.redhat.com/browse/DPTP-2871 # The configured job timeout after isn't signaling the test script like it @@ -223,23 +226,14 @@ case "${CLOUD}" in ;; "vsphere") BASE_DOMAIN="${BASE_DOMAIN:-vmc.devcluster.openshift.com}" - if [ -z "$NETWORK_NAME" ]; then - echo "Variable 'NETWORK_NAME' not set." - exit 1 - fi - if [ -z "$VCENTER" ]; then - echo "Variable 'VCENTER' not set." + USE_MANAGED_DNS=false + if [ -z "$VSPHERE_INSTALLER_PLATFORM_SPEC_JSON" ]; then + echo "Variable 'VSPHERE_INSTALLER_PLATFORM_SPEC_JSON' not set." exit 1 fi - API_VIP=$(get_vips 3) # Get 3rd vip from file - INGRESS_VIP=$(get_vips 4) # Get 4th vip from file - EXTRA_CREATE_CLUSTER_ARGS="--vsphere-datacenter=${GOVC_DATACENTER:-DEVQEdatacenter} \ - --vsphere-default-datastore=${GOVC_DATASTORE:-vsanDatastore}\ - --vsphere-cluster=${VSPHERE_CLUSTER:-DEVQEcluster} - --vsphere-api-vip=$API_VIP \ - --vsphere-ingress-vip=$INGRESS_VIP \ - --vsphere-network=$NETWORK_NAME \ - --vsphere-vcenter=$VCENTER" + EXTRA_CREATE_CLUSTER_ARGS="--machine-network=$VSPHERE_MACHINE_NETWORK \ + --vsphere-api-vip=$VSPHERE_API_VIP \ + --vsphere-ingress-vip=$VSPHERE_INGRESS_VIP" ;; *) echo "unknown cloud: ${CLOUD}" @@ -317,22 +311,4 @@ function capture_cluster_logs() { ${SRC_ROOT}/hack/logextractor.sh ${CLUSTER_NAME} "${ARTIFACT_DIR}/hive" exit 1 fi -} - -function get_vips() { - # Return vip at given index - idx=${1:-1} - if [ -z "$SHARED_DIR" ]; then - echo "Variable 'SHARED_DIR' not set." - exit 1 - fi - - vips="${SHARED_DIR}/vips.txt" - if [ ! -f "$vips" ]; then - echo "Error: File '$vips' not found." - exit 1 - fi - - vip=$(sed -n "${idx}p" "$vips") - echo "$vip" } \ No newline at end of file diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 8ca7c8f67b9..f468239b2bc 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -335,6 +335,9 @@ const ( // VSphereDataStoreEnvVar is the environment variable specifying the vSphere default datastore. VSphereDataStoreEnvVar = "GOVC_DATASTORE" + // VSpherePlatformSpecJSONEnvVar is the environment variable containing an installer vSphere platform spec, encoded as JSON + VSpherePlatformSpecJSONEnvVar = "VSPHERE_INSTALLER_PLATFORM_SPEC_JSON" + // VSphereCredentialsDir is the directory containing VSphere credentials files. VSphereCredentialsDir = "/vsphere-credentials"