terraformvsphereterraform-provider-vsphere

Create multiple new instances with Terraform

I have written a Terraform script which is supposed to create multiple server instances from one template. I have now created two different variable files.

But when I run the script, a new instance is created with the first variable file, but with the second the first is always overwritten / changed. I don't know why Terraform is referencing the previously newly created instance. How can I prevent this?

server-1.tfvars:

vsphere_user                     = "administrator@vsphere.local"
vsphere_password                 = "#Password"
vsphere_server                   = "vsphere.server"
vsphere_datacenter               = "Datacenter"
vsphere_datastore                = "Storage_1"
vsphere_compute_cluster          = "Cluster"
vsphere_network                  = "Network_1"
vsphere_virtual_machine_template = "Template_Microsoft_Windows_Server_2019_x64_english"
system_name                      = "server-1"
system_cores                     = 2
system_cores_per_socket          = 2
system_memory                    = 2048
system_local_admin_password      = "#Password"
system_ipv4_address              = "172.22.15.11"
system_ipv4_netmask              = 24
system_dns_server_list           = ["172.22.15.101"]
system_ipv4_gateway              = "172.22.15.1"
system_disk1_size                = 75
system_domain_admin_user         = "Administrator"
system_domain_admin_password     = "#Password"

server-2.tfvars:

vsphere_user                     = "administrator@vsphere.local"
vsphere_password                 = "#Password"
vsphere_server                   = "vsphere.server"
vsphere_datacenter               = "Datacenter"
vsphere_datastore                = "Storage_1"
vsphere_compute_cluster          = "Cluster"
vsphere_network                  = "Network_1"
vsphere_virtual_machine_template = "Template_Microsoft_Windows_Server_2019_x64_english"
system_name                      = "server-2"
system_cores                     = 2
system_cores_per_socket          = 2
system_memory                    = 2048
system_local_admin_password      = "#Password"
system_ipv4_address              = "172.22.15.12"
system_ipv4_netmask              = 24
system_dns_server_list           = ["172.22.15.101"]
system_ipv4_gateway              = "172.22.15.1"
system_disk1_size                = 75
system_domain_admin_user         = "Administrator"
system_domain_admin_password     = "#Password"

provider.tf:

provider "vsphere" {
  user                 = var.vsphere_user
  password             = var.vsphere_password
  vsphere_server       = var.vsphere_server
  allow_unverified_ssl = true
}

data.tf:

# Data Sources
# Datacenter
data "vsphere_datacenter" "dc" {
  name = var.vsphere_datacenter
}

# Datastore
data "vsphere_datastore" "datastore" {
  name          = var.vsphere_datastore
  datacenter_id = data.vsphere_datacenter.dc.id
}

# Cluster
data "vsphere_compute_cluster" "cluster" {
  name          = var.vsphere_compute_cluster
  datacenter_id = data.vsphere_datacenter.dc.id
}

# Network
data "vsphere_network" "network" {
  name          = var.vsphere_network
  datacenter_id = data.vsphere_datacenter.dc.id
}

# Template
data "vsphere_virtual_machine" "template" {
  name          = var.vsphere_virtual_machine_template
  datacenter_id = data.vsphere_datacenter.dc.id
}

resource.tf:

# Virtual Machine Resource
resource "vsphere_virtual_machine" "server-instance" {
  # System
  firmware  = "efi"
  guest_id  = data.vsphere_virtual_machine.template.guest_id
  scsi_type = data.vsphere_virtual_machine.template.scsi_type

  # VM-Name
  name             = var.system_name
  resource_pool_id = data.vsphere_compute_cluster.cluster.resource_pool_id
  datastore_id     = data.vsphere_datastore.datastore.id

  # CPU
  num_cpus               = var.system_cores
  num_cores_per_socket   = var.system_cores_per_socket
  cpu_hot_add_enabled    = true
  cpu_hot_remove_enabled = true

  # Memory
  memory                 = var.system_memory
  memory_hot_add_enabled = true

  # Network
  network_interface {
    network_id   = data.vsphere_network.network.id
    adapter_type = "e1000e"
  }

  # Storage
  # Drive 0 (C)
  disk {
    label            = "disk0"
    unit_number      = 0
    size             = data.vsphere_virtual_machine.template.disks.0.size
    eagerly_scrub    = data.vsphere_virtual_machine.template.disks.0.eagerly_scrub
    thin_provisioned = data.vsphere_virtual_machine.template.disks.0.thin_provisioned
  }

  # Drive 1 (D)
  disk {
    label            = "disk1"
    unit_number      = 1
    size             = var.system_disk1_size
    eagerly_scrub    = data.vsphere_virtual_machine.template.disks.1.eagerly_scrub
    thin_provisioned = data.vsphere_virtual_machine.template.disks.1.thin_provisioned
  }

  # Template clone and OS settings
  clone {
    template_uuid = data.vsphere_virtual_machine.template.id

    customize {
      windows_options {
        computer_name         = var.system_name
        admin_password        = random_password.password.result
        join_domain           = var.system_domain
        domain_admin_user     = var.system_domain_admin_user
        domain_admin_password = var.system_domain_admin_password
        auto_logon            = true
      }

      network_interface {
        ipv4_address    = var.system_ipv4_address
        ipv4_netmask    = var.system_ipv4_netmask
        dns_server_list = var.system_dns_server_list
      }

      ipv4_gateway = var.system_ipv4_gateway
    }
  }
}

password.tf:

# Import the Random Password Provider
terraform {
  required_providers {
    random = {
      source = "hashicorp/random"
    }
  }
}

resource "random_password" "password" {
  length           = 25
  upper            = true
  lower            = true
  number           = true
  special          = true
  min_upper        = 2
  min_lower        = 2
  min_numeric      = 2
  min_special      = 1
  override_special = "!@#$%&*()-_=+[]{}<>:?"
}
Solution

  • Terraform's model is that each resource instance in your configuration is bound to zero or one remote objects -- zero if you've not created a remote object yet, and then one after the object has been created for the first time.

    The other important characteristic of Terraform's model is that it's declarative. You shouldn't understand terraform apply as "create all of these things", but rather as "take whatever actions are needed to make the remote system match this configuration". On the first run that will typically result in a number of create actions, but on subsequent runs you'll usually be making changes to objects that already exist, because the provider aims to find the least disruptive way to change the remote system to match the updated configuration.

    With that in mind, when you re-run Terraform with different values for the arguments of vsphere_virtual_machine.server-instance, Terraform (and the vsphere provider) understands that as you wanting to change the existing object that you created earlier, not to create a new object.

    In order to have multiple virtual machines existing at once you must have multiple corresponding resource instances, with one for each of your virtual machines. In Terraform's model, each resource block can represent one or more resource instances; the most straightforward way to get two resource instances is to write two resource blocks, which will therefore declare one instance each:

    resource "vsphere_virtual_machine" "server_1" {
  # ...
}

resource "vsphere_virtual_machine" "server_2" {
  # ...
}

    However, if your multiple instances are created systematically in a way that you can express using expressions in the Terraform language, then you have some other options.

    If all you consider all of your virtual machines to in some sense be "copies" of one another, all functionally equivalent, then you might choose to use the count meta-argument, which causes a resource block to have multiple resource instances associated with it -- the number given by the count expression -- that all have largely the same configuration aside from some minor differences expressed in terms of the special symbol count.index, which gives the index of the current instance:

    resource "vsphere_virtual_machine" "server" {
  count = 2

  # VM-Name
  name             = "${var.system_name}-${count.index}"
  resource_pool_id = data.vsphere_compute_cluster.cluster.resource_pool_id
  datastore_id     = data.vsphere_datastore.datastore.id

  # System
  firmware  = "efi"
  guest_id  = data.vsphere_virtual_machine.template.guest_id
  scsi_type = data.vsphere_virtual_machine.template.scsi_type

  # CPU
  num_cpus               = var.system_cores
  num_cores_per_socket   = var.system_cores_per_socket
  cpu_hot_add_enabled    = true
  cpu_hot_remove_enabled = true

  # Memory
  memory                 = var.system_memory
  memory_hot_add_enabled = true

  # Network
  network_interface {
    network_id   = data.vsphere_network.network.id
    adapter_type = "e1000e"
  }

  # Storage
  # Drive 0 (C)
  disk {
    label            = "disk0"
    unit_number      = 0
    size             = data.vsphere_virtual_machine.template.disks.0.size
    eagerly_scrub    = data.vsphere_virtual_machine.template.disks.0.eagerly_scrub
    thin_provisioned = data.vsphere_virtual_machine.template.disks.0.thin_provisioned
  }

  # Drive 1 (D)
  disk {
    label            = "disk1"
    unit_number      = 1
    size             = var.system_disk1_size
    eagerly_scrub    = data.vsphere_virtual_machine.template.disks.1.eagerly_scrub
    thin_provisioned = data.vsphere_virtual_machine.template.disks.1.thin_provisioned
  }

  # Template clone and OS settings
  clone {
    template_uuid = data.vsphere_virtual_machine.template.id

    customize {
      windows_options {
        computer_name         = var.system_name
        admin_password        = random_password.password.result
        join_domain           = var.system_domain
        domain_admin_user     = var.system_domain_admin_user
        domain_admin_password = var.system_domain_admin_password
        auto_logon            = true
      }

      network_interface {
        ipv4_address    = var.system_ipv4_address
        ipv4_netmask    = var.system_ipv4_netmask
        dns_server_list = var.system_dns_server_list
      }

      ipv4_gateway = var.system_ipv4_gateway
    }
  }
}

    The above is the same as the resource "vsphere_virtual_machine" "server" you included except that I added count = 2 at the start and I changed name so that it considers var.system_name to be a name prefix rather than a whole name, adding the current index in order to create a complete unique name. I imagine you'd probably also need to follow a similar strategy for system_ipv4_address, possibly using the cidrhost function to systematically calculate IP addresses, but I'll leave that out here for simplicity's sake.

    If we also change the variables file so that var.system_name is just "server" rather than "server-1" then this would declare two resource instances from the single resource block:

    In the examples you shared it seems like count would be the best fit for your situation because your server virtual machines are otherwise all configured the same. However, if you instead need to treat each server as configured totally independently, so that they can all have potentially different arguments, then you have another option in the form of the for_each meta-argument. As with count it declares multiple resource instances from a single resource block, but it does so for each element in a map rather than just for increasing integers up to a particular limit.

    This approach does require a slightly different strategy for the input variables, because we need the input to be a map of objects where each element of the map represents one virtual machine:

    variable "virtual_machines" {
  type = map(object({
    system_cores            = number
    system_cores_per_socket = number
    system_memory           = number
    system_ipv4_address     = string
    # (and so on, for all of the attributes that vary between
    # your virtual machines)
  }))
}

    Because this is a single variable defining all of the virtual machines, you'll also need to change your .tfvars file to set it in a different way:

    vsphere_user                     = "administrator@vsphere.local"
vsphere_password                 = "#Password"
vsphere_server                   = "vsphere.server"
vsphere_datacenter               = "Datacenter"
vsphere_datastore                = "Storage_1"
vsphere_compute_cluster          = "Cluster"
vsphere_network                  = "Network_1"
vsphere_virtual_machine_template = "Template_Microsoft_Windows_Server_2019_x64_english"

virtual_machines = {
  server-1 = {
    system_cores            = 2
    system_cores_per_socket = 2
    system_memory           = 2048
    system_ipv4_address     = "172.22.15.11"
    # ...
  }
  server-2 = {
    system_cores            = 2
    system_cores_per_socket = 2
    system_memory           = 2048
    system_ipv4_address     = "172.22.15.12"
    # ...
  }
}

    The resource block with for_each set would then look something like this:

    resource "vsphere_virtual_machine" "server" {
  for_each = var.virtual_machines

  # VM-Name
  name             = each.key
  resource_pool_id = data.vsphere_compute_cluster.cluster.resource_pool_id
  datastore_id     = data.vsphere_datastore.datastore.id

  # System
  firmware  = "efi"
  guest_id  = data.vsphere_virtual_machine.template.guest_id
  scsi_type = data.vsphere_virtual_machine.template.scsi_type

  # CPU
  num_cpus               = each.value.system_cores
  num_cores_per_socket   = each.value.system_cores_per_socket
  cpu_hot_add_enabled    = true
  cpu_hot_remove_enabled = true

  # Memory
  memory                 = each.value.system_memory
  memory_hot_add_enabled = true

  # Network
  network_interface {
    network_id   = data.vsphere_network.network.id
    adapter_type = "e1000e"
  }

  # Storage
  # Drive 0 (C)
  disk {
    label            = "disk0"
    unit_number      = 0
    size             = data.vsphere_virtual_machine.template.disks.0.size
    eagerly_scrub    = data.vsphere_virtual_machine.template.disks.0.eagerly_scrub
    thin_provisioned = data.vsphere_virtual_machine.template.disks.0.thin_provisioned
  }

  # Drive 1 (D)
  disk {
    label            = "disk1"
    unit_number      = 1
    size             = each.value.system_disk1_size
    eagerly_scrub    = data.vsphere_virtual_machine.template.disks.1.eagerly_scrub
    thin_provisioned = data.vsphere_virtual_machine.template.disks.1.thin_provisioned
  }

  # Template clone and OS settings
  clone {
    template_uuid = data.vsphere_virtual_machine.template.id

    customize {
      windows_options {
        computer_name         = each.value.system_name
        admin_password        = random_password.password.result
        join_domain           = each.value.system_domain
        domain_admin_user     = each.value.system_domain_admin_user
        domain_admin_password = each.value.system_domain_admin_password
        auto_logon            = true
      }

      network_interface {
        ipv4_address    = each.value.system_ipv4_address
        ipv4_netmask    = each.value.system_ipv4_netmask
        dns_server_list = each.value.system_dns_server_list
      }

      ipv4_gateway = each.value.system_ipv4_gateway
    }
  }
}

    Again this is mostly the same as your original resource block, but I've added for_each = var.virtual_machines, set the name to each.key to use the map keys as names, and replaced all of the other references to variables with references to attributes of each.value, which represents the value of the current element from the map.

    In this case, this resource block will declare the following resource instances:

    Notice that Terraform is now using the map key to identify each instance, and so if you edit a value associated with an existing key in the .tfvars file and run terraform plan then Terraform will understand that as you intending to update the existing object with that key, but if you add an entirely new key to the map then Terraform will understand that as you intending to create a new virtual machine. Over time you can update, create, and delete virtual machines by update, creating, and deleting their corresponding entries in var.virtual_machines.

    Terraform expects that each time you run it you are providing it with a description of the full state of the part of the system it's responsible for managing, so there isn't a usage model where you can just ask Terraform to add a new virtual machine without also providing an unchanged configuration for all of the existing ones. If you omit the existing ones then Terraform will understand that you intend to destroy them.