feat: add support for datastore clusters

Adds support for the use of datastore clusters in the applicable builders and post-processors.

Signed-off-by: Ryan Johnson <ryan@tenthirtyam.org>

Author: tenthirtyam

.web-docs/components/builder/vsphere-clone/README.md

@@ -1036,7 +1036,15 @@ wget http://{{ .HTTPIP }}:{{ .HTTPPort }}/foo/bar/preseed.cfg
   a nested path might resemble 'rp-packer/rp-linux-images'.
 
 - `datastore` (string) - The datastore where the virtual machine is created.
-  Required if `host` is a cluster, or if `host` has multiple datastores.
+  Required if `host` is a cluster or if `host` has multiple datastores,
+  unless `datastore_cluster` is specified.
+  
+  ~> **Note:** Cannot be used with `datastore_cluster`.
+
+- `datastore_cluster` (string) - The datastore cluster where the virtual machine is created.
+  When specified, Storage DRS will automatically select the optimal datastore.
+  
+  ~> **Note:** Cannot be used with `datastore`.
 
 - `set_host_for_datastore_uploads` (bool) - The ESXI host used for uploading files to the datastore.
   Defaults to `false`.

.web-docs/components/builder/vsphere-iso/README.md

@@ -156,7 +156,15 @@ wget http://{{ .HTTPIP }}:{{ .HTTPPort }}/foo/bar/preseed.cfg
   a nested path might resemble 'rp-packer/rp-linux-images'.
 
 - `datastore` (string) - The datastore where the virtual machine is created.
-  Required if `host` is a cluster, or if `host` has multiple datastores.
+  Required if `host` is a cluster or if `host` has multiple datastores,
+  unless `datastore_cluster` is specified.
+  
+  ~> **Note:** Cannot be used with `datastore_cluster`.
+
+- `datastore_cluster` (string) - The datastore cluster where the virtual machine is created.
+  When specified, Storage DRS will automatically select the optimal datastore.
+  
+  ~> **Note:** Cannot be used with `datastore`.
 
 - `set_host_for_datastore_uploads` (bool) - The ESXI host used for uploading files to the datastore.
   Defaults to `false`.

.web-docs/components/post-processor/vsphere/README.md

@@ -30,8 +30,6 @@ The following configuration options are available for the post-processor.
 - `datacenter` (string) - The name of the vSphere datacenter object to place the virtual machine.
   This is _not required_ if `resource_pool` is specified.
 
-- `datastore` (string) - The name of the vSphere datastore to place the virtual machine.
-
 - `host` (string) - The fully qualified domain name or IP address of the vCenter instance or ESX host.
 
 - `password` (string) - The password to use to authenticate to the vSphere endpoint.
@@ -45,6 +43,13 @@ The following configuration options are available for the post-processor.
 
 <!-- Code generated from the comments of the Config struct in post-processor/vsphere/post-processor.go; DO NOT EDIT MANUALLY -->
 
+- `datastore` (string) - The name of the vSphere datastore to place the virtual machine.
+  Mutually exclusive with `datastore_cluster`.
+
+- `datastore_cluster` (string) - The name of the vSphere datastore cluster to place the virtual machine.
+  When specified, Storage DRS will automatically select the optimal datastore.
+  Mutually exclusive with `datastore`.
+
 - `disk_mode` (string) - The disk format of the target virtual machine. One of `thin`, `thick`,
 
 - `esxi_host` (string) - The fully qualified domain name or IP address of the ESX host to upload the
@@ -55,7 +60,8 @@ The following configuration options are available for the post-processor.
 - `options` ([]string) - Options to send to `ovftool` when uploading the virtual machine.
   Use `ovftool --help` to list all the options available.
 
-- `overwrite` (bool) - Overwrite existing files. Defaults to `false`.
+- `overwrite` (bool) - Overwrite existing files.
+  If `true`, forces overwrites of existing files. Defaults to `false`.
 
 - `resource_pool` (string) - The name of the resource pool to place the virtual machine.
 

GNUmakefile

@@ -6,6 +6,9 @@ COUNT?=1
 TEST?=$(shell go list ./...)
 HASHICORP_PACKER_PLUGIN_SDK_VERSION?=$(shell go list -m github.com/hashicorp/packer-plugin-sdk | cut -d " " -f2)
 
+# Use Go 1.23.12 specifically for packer-sdc related commands
+GO_SDC ?= env GOTOOLCHAIN=go1.23.12 go
+
 .PHONY: dev
 
 build:
@@ -19,7 +22,7 @@ test:
 	@go test -race -count $(COUNT) $(TEST) -timeout=3m
 
 install-packer-sdc: ## Install packer sofware development command
-	@go install github.com/hashicorp/packer-plugin-sdk/cmd/packer-sdc@${HASHICORP_PACKER_PLUGIN_SDK_VERSION}
+	@$(GO_SDC) install github.com/hashicorp/packer-plugin-sdk/cmd/packer-sdc@${HASHICORP_PACKER_PLUGIN_SDK_VERSION}
 
 plugin-check: install-packer-sdc build
 	@packer-sdc plugin-check ${BINARY}
@@ -28,7 +31,7 @@ testacc: dev
 	@PACKER_ACC=1 go test -count $(COUNT) -v $(TEST) -timeout=120m
 
 generate: install-packer-sdc
-	@go generate ./...
+	@$(GO_SDC) generate ./...
 	@rm -rf .docs
 	@packer-sdc renderdocs -src "docs" -partials docs-partials/ -dst ".docs/"
 	@./.web-docs/scripts/compile-to-webdocs.sh "." ".docs" ".web-docs" "hashicorp"

builder/vsphere/clone/builder.go

@@ -44,6 +44,11 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 		&common.StepConnect{
 			Config: &b.config.ConnectConfig,
 		},
+		&common.StepResolveDatastore{
+			Datastore:        b.config.Datastore,
+			DatastoreCluster: b.config.DatastoreCluster,
+			DiskCount:        len(b.config.StorageConfig.Storage),
+		},
 		&commonsteps.StepCreateCD{
 			Files:   b.config.CDFiles,
 			Content: b.config.CDContent,

builder/vsphere/clone/config.hcl2spec.go

@@ -9,6 +9,7 @@ package clone
 import (
 	"context"
 	"fmt"
+	"log"
 	"path"
 	"strings"
 
@@ -17,6 +18,7 @@ import (
 	"github.com/hashicorp/packer-plugin-sdk/packerbuilderdata"
 	"github.com/hashicorp/packer-plugin-vsphere/builder/vsphere/common"
 	"github.com/hashicorp/packer-plugin-vsphere/builder/vsphere/driver"
+	"github.com/vmware/govmomi/vim25/types"
 )
 
 type vAppConfig struct {
@@ -159,13 +161,61 @@ func (s *StepCloneVM) Run(ctx context.Context, state multistep.StateBag) multist
 		})
 	}
 
+	datastoreName := s.Location.Datastore
+	var primaryDatastore driver.Datastore
+	if ds, ok := state.GetOk("datastore"); ok {
+		primaryDatastore = ds.(driver.Datastore)
+		datastoreName = primaryDatastore.Name()
+	}
+
+	// If no datastore was resolved and no datastore was specified, return an error
+	if datastoreName == "" && s.Location.DatastoreCluster == "" {
+		state.Put("error", fmt.Errorf("no datastore specified and no datastore resolved from cluster"))
+		return multistep.ActionHalt
+	}
+
+	// Handle multi-disk placement when using a datastore cluster.
+	var datastoreRefs []*types.ManagedObjectReference
+	if s.Location.DatastoreCluster != "" && len(disks) > 1 {
+		if vcDriver, ok := d.(*driver.VCenterDriver); ok {
+			// Request Storage DRS recommendations for all disks at once for optimal placement.
+			ui.Sayf("Requesting Storage DRS recommendations for %d disks...", len(disks))
+
+			diskDatastores, method, err := vcDriver.SelectDatastoresForDisks(s.Location.DatastoreCluster, disks)
+			if err != nil {
+				ui.Errorf("Warning: Failed to get Storage DRS recommendations: %s. Using primary datastore.", err)
+				if primaryDatastore != nil {
+					ref := primaryDatastore.Reference()
+					for i := 0; i < len(disks); i++ {
+						datastoreRefs = append(datastoreRefs, &ref)
+					}
+				}
+			} else {
+				// Use the first disk's datastore as the primary datastore.
+				if len(diskDatastores) > 0 {
+					datastoreName = diskDatastores[0].Name()
+				}
+
+				for i, ds := range diskDatastores {
+					ref := ds.Reference()
+					if method == driver.SelectionMethodDRS {
+						log.Printf("[INFO] Disk %d: Storage DRS selected datastore '%s'", i+1, ds.Name())
+					} else {
+						log.Printf("[INFO] Disk %d: Using first available datastore '%s'", i+1, ds.Name())
+					}
+					datastoreRefs = append(datastoreRefs, &ref)
+				}
+			}
+		}
+	}
+
 	vm, err := template.Clone(ctx, &driver.CloneConfig{
 		Name:            s.Location.VMName,
 		Folder:          s.Location.Folder,
 		Cluster:         s.Location.Cluster,
 		Host:            s.Location.Host,
 		ResourcePool:    s.Location.ResourcePool,
-		Datastore:       s.Location.Datastore,
+		Datastore:       datastoreName,
 		LinkedClone:     s.Config.LinkedClone,
 		Network:         s.Config.Network,
 		MacAddress:      strings.ToLower(s.Config.MacAddress),
@@ -175,6 +225,7 @@ func (s *StepCloneVM) Run(ctx context.Context, state multistep.StateBag) multist
 		StorageConfig: driver.StorageConfig{
 			DiskControllerType: s.Config.StorageConfig.DiskControllerType,
 			Storage:            disks,
+			DatastoreRefs:      datastoreRefs,
 		},
 	})
 	if err != nil {

builder/vsphere/clone/step_clone.go

@@ -35,8 +35,16 @@ type LocationConfig struct {
 	// a nested path might resemble 'rp-packer/rp-linux-images'.
 	ResourcePool string `mapstructure:"resource_pool"`
 	// The datastore where the virtual machine is created.
-	// Required if `host` is a cluster, or if `host` has multiple datastores.
+	// Required if `host` is a cluster or if `host` has multiple datastores,
+	// unless `datastore_cluster` is specified.
+	//
+	// ~> **Note:** Cannot be used with `datastore_cluster`.
 	Datastore string `mapstructure:"datastore"`
+	// The datastore cluster where the virtual machine is created.
+	// When specified, Storage DRS will automatically select the optimal datastore.
+	//
+	// ~> **Note:** Cannot be used with `datastore`.
+	DatastoreCluster string `mapstructure:"datastore_cluster"`
 	// The ESXI host used for uploading files to the datastore.
 	// Defaults to `false`.
 	SetHostForDatastoreUploads bool `mapstructure:"set_host_for_datastore_uploads"`
@@ -52,7 +60,10 @@ func (c *LocationConfig) Prepare() []error {
 		errs = append(errs, fmt.Errorf("'host' or 'cluster' is required"))
 	}
 
-	// clean Folder path and remove leading slash as folders are relative within vsphere
+	if c.Datastore != "" && c.DatastoreCluster != "" {
+		errs = append(errs, fmt.Errorf("'datastore' and 'datastore_cluster' are mutually exclusive; specify only one"))
+	}
+
 	c.Folder = path.Clean(c.Folder)
 	c.Folder = strings.TrimLeft(c.Folder, "/")
 

builder/vsphere/common/config_location.go

@@ -63,10 +63,18 @@ func (s *StepAddFloppy) Run(_ context.Context, state multistep.StateBag) multist
 	if floppyPath, ok := state.GetOk("floppy_path"); ok {
 		ui.Say("Uploading floppy image...")
 
-		ds, err := d.FindDatastore(s.Datastore, s.Host)
-		if err != nil {
-			state.Put("error", err)
-			return multistep.ActionHalt
+		var ds driver.Datastore
+		var err error
+
+		// If a datastore was resolved (from datastore or datastore_cluster), use it.
+		if resolvedDs, ok := state.GetOk("datastore"); ok {
+			ds = resolvedDs.(driver.Datastore)
+		} else {
+			ds, err = d.FindDatastore(s.Datastore, s.Host)
+			if err != nil {
+				state.Put("error", err)
+				return multistep.ActionHalt
+			}
 		}
 		vmDir, err := vm.GetDir()
 		if err != nil {
@@ -123,10 +131,18 @@ func (s *StepAddFloppy) Cleanup(state multistep.StateBag) {
 	if UploadedFloppyPath, ok := state.GetOk("uploaded_floppy_path"); ok {
 		ui.Say("Deleting floppy image...")
 
-		ds, err := d.FindDatastore(s.Datastore, s.Host)
-		if err != nil {
-			state.Put("error", err)
-			return
+		var ds driver.Datastore
+		var err error
+
+		// If a datastore was resolved (from datastore or datastore_cluster), use it.
+		if resolvedDs, ok := state.GetOk("datastore"); ok {
+			ds = resolvedDs.(driver.Datastore)
+		} else {
+			ds, err = d.FindDatastore(s.Datastore, s.Host)
+			if err != nil {
+				state.Put("error", err)
+				return
+			}
 		}
 
 		err = ds.Delete(UploadedFloppyPath.(string))